diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd15dcadea..3b70fc7263 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,11 @@ jobs: run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse-dev - name: Build run: | - go build ./... + CGO_ENABLED=0 go build ./... go install ./tools/build_gcsfuse build_gcsfuse . /tmp ${GITHUB_SHA} - name: Test - run: go test -p 1 -count 1 -v -cover ./... + run: CGO_ENABLED=0 go test -p 1 -count 1 -v -cover ./... lint: name: Lint runs-on: ubuntu-20.04 diff --git a/Dockerfile b/Dockerfile index cb12d822db..62dd0aee4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse -FROM golang:1.20.4-alpine as builder +FROM golang:1.20.5-alpine as builder RUN apk add git diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 4015dd11a2..eebc50746a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -6,7 +6,7 @@ This page enumerates some common user facing issues around GCSFuse and also disc | Generic Mounting Issue | Most of the common mount point issues are around permissions on both local mount point and the Cloud Storage bucket. It is highly recommended to retry with --foreground --debug_fuse --debug_fs --debug_gcs --debug_http flags which would provide much more detailed logs to understand the errors better and possibly provide a solution. | | Mount successful but files not visible | Try mounting the gcsfuse with --implicit-dir flag. Read the [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md) to know the reasoning. | | Mount failed with fusermount3 exit status 1 | It comes when the bucket is already mounted in a folder and we try to mount it again. You need to unmount first and then remount. | -| Mount failed with error: Current requires cgo or $USER set in environment | It comes when we try mounting by building the gcsfuse codebase. To fix this, build the gcsfuse package by enabling the CGO_ENABLED flag in the go env and then mount back.
  1. Check the current value using - ```go env``` command.
  2. If it is unset, set this using - ```export CGO_ENABLED=1``` command.
| +| version `GLIBC_x.yz` not found | GCSFuse should not be linking to glibc. Please either `export CGO_ENABLED=0` in your environment or prefix `CGO_ENABLED=0` to any `go build\|run\|test` commands that you're invoking. | | Mount get stuck with error: DefaultTokenSource: google: could not find default credentials | Run ```gcloud auth application-default login``` command to fetch default credentials to the VM. This will fetch the credentials to the following locations:
  1. For linux - $HOME/.config/gcloud/application_default_credentials.json
  2. For windows - %APPDATA%/gcloud/applicateion_default_credentials.json
| | Input/Output Error | It’s a generic error, but the most probable culprit is the bucket not having the right permission for Cloud Storage FUSE to operate on. Ref - [here](https://stackoverflow.com/questions/36382704/gcsfuse-input-output-error) | | Generic NO_PUBKEY Error - while installing Cloud Storage FUSE on ubuntu 22.04 | It happens while running - ```sudo apt-get update``` - working on installing Cloud Storage FUSE. You just have to add the pubkey you get in the error using the below command: ```sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys ``` And then try running ```sudo apt-get update``` | diff --git a/internal/perms/perms.go b/internal/perms/perms.go index 42834a2c9b..067469bebd 100644 --- a/internal/perms/perms.go +++ b/internal/perms/perms.go @@ -17,35 +17,23 @@ package perms import ( "fmt" - "os/user" - "strconv" + "os" ) // MyUserAndGroup returns the UID and GID of this process. -func MyUserAndGroup() (uid uint32, gid uint32, err error) { - // Ask for the current user. - user, err := user.Current() - if err != nil { - err = fmt.Errorf("Fetching current user: %w", err) - return - } - - // Parse UID. - uid64, err := strconv.ParseUint(user.Uid, 10, 32) - if err != nil { - err = fmt.Errorf("Parsing UID (%s): %w", user.Uid, err) - return - } +func MyUserAndGroup() (uid, gid uint32, err error) { + // An improvement idea is to invoke os.current.User() and use its partial output + // even when the call itself returned error. + signed_uid := os.Getuid() + signed_gid := os.Getgid() - // Parse GID. - gid64, err := strconv.ParseUint(user.Gid, 10, 32) - if err != nil { - err = fmt.Errorf("Parsing GID (%s): %w", user.Gid, err) + if signed_gid == -1 || signed_uid == -1 { + err = fmt.Errorf("failed to get uid/gid. UID = %d, GID = %d", signed_uid, signed_gid) return } - uid = uint32(uid64) - gid = uint32(gid64) + uid = uint32(signed_uid) + gid = uint32(signed_gid) return } diff --git a/internal/perms/perms_test.go b/internal/perms/perms_test.go new file mode 100644 index 0000000000..217fbd9cb0 --- /dev/null +++ b/internal/perms/perms_test.go @@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// System permissions-related code unit tests. +package perms_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/internal/perms" + . "github.com/jacobsa/ogletest" +) + +func TestPerms(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type PermsTest struct { +} + +func init() { RegisterTestSuite(&PermsTest{}) } + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *PermsTest) MyUserAndGroupNoError() { + uid, gid, err := perms.MyUserAndGroup() + ExpectEq(err, nil) + + unexpected_id_signed := -1 + unexpected_id := uint32(unexpected_id_signed) + ExpectNe(uid, unexpected_id) + ExpectNe(gid, unexpected_id) +} diff --git a/perfmetrics/scripts/compare_fuse_types_using_fio.py b/perfmetrics/scripts/compare_fuse_types_using_fio.py index 513baa6b31..bb1ad1d7a2 100644 --- a/perfmetrics/scripts/compare_fuse_types_using_fio.py +++ b/perfmetrics/scripts/compare_fuse_types_using_fio.py @@ -61,7 +61,7 @@ def _install_gcsfuse_source(gcs_bucket, gcsfuse_flags) -> None: os.system(f'''git clone {GCSFUSE_REPO} mkdir gcs cd gcsfuse - go run . {gcsfuse_flags} {gcs_bucket} ../gcs + CGO_ENABLED=0 go run . {gcsfuse_flags} {gcs_bucket} ../gcs cd .. ''') diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index 3886b08a20..324c9207a9 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -6,8 +6,8 @@ echo "Installing git" sudo apt-get install git echo "Installing pip" sudo apt-get install pip -y -echo "Installing go-lang 1.20.4" -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +echo "Installing go-lang 1.20.5" +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin echo "Installing fio" @@ -28,7 +28,7 @@ commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) git checkout $commitId echo "Executing integration tests" -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test +GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test # Checkout back to master branch to use latest CI test scripts in master. git checkout master diff --git a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh index 1af6bd1f55..be82b16994 100644 --- a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh @@ -1,13 +1,14 @@ #!/bin/bash -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +# Install golang +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz export PATH=$PATH:/usr/local/go/bin # Clone and build the gcsfuse master branch. git clone https://github.com/GoogleCloudPlatform/gcsfuse.git cd gcsfuse -go build . +CGO_ENABLED=0 go build . cd - # Create a directory for gcsfuse logs diff --git a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py index 9ceaf7e5ca..34675cccc3 100644 --- a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py +++ b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py @@ -105,7 +105,7 @@ def _run_from_source(gcs_bucket, data_directory_name) -> None: os.system(f'''mkdir {data_directory_name} git clone {GITHUB_REPO} cd gcsfuse - go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name} + CGO_ENABLED=0 go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name} cd .. ''') diff --git a/perfmetrics/scripts/ml_tests/setup.sh b/perfmetrics/scripts/ml_tests/setup.sh index f0990cdd63..0f0987f6ee 100644 --- a/perfmetrics/scripts/ml_tests/setup.sh +++ b/perfmetrics/scripts/ml_tests/setup.sh @@ -4,7 +4,7 @@ # >> source setup.sh # Go version to be installed. -GO_VERSION=go1.20.4.linux-amd64.tar.gz +GO_VERSION=go1.20.5.linux-amd64.tar.gz # This function will install the given module/dependency if it's not alredy # installed. diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh index 1d6ec891cc..f6299d6b61 100644 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh @@ -5,14 +5,14 @@ # and epochs functionality, and runs the model # Install go lang -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin # Clone the repo and build gcsfuse git clone "https://github.com/GoogleCloudPlatform/gcsfuse.git" cd gcsfuse -go build . +CGO_ENABLED=0 go build . cd - # Mount the bucket and run in background so that docker doesn't keep running after resnet_runner.py fails diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 49072ef6ef..9720b1b434 100644 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -22,8 +22,8 @@ pip install google-cloud pip install google-cloud-vision pip install google-api-python-client pip install prettytable -echo Installing go-lang 1.20.4 -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +echo Installing go-lang 1.20.5 +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin echo Installing fio @@ -38,7 +38,7 @@ GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100" BUCKET_NAME=presubmit-perf-tests MOUNT_POINT=gcs # The VM will itself exit if the gcsfuse mount fails. -go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT +CGO_ENABLED=0 go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT touch result.txt # Running FIO test chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh @@ -53,13 +53,13 @@ echo checkout PR branch git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER # Executing integration tests -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test +GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test # Executing perf tests echo Mounting gcs bucket from pr branch mkdir -p gcs # The VM will itself exit if the gcsfuse mount fails. -go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT +CGO_ENABLED=0 go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT # Running FIO test chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh diff --git a/tools/build_gcsfuse/main.go b/tools/build_gcsfuse/main.go index bb52ea1191..b1a4022481 100644 --- a/tools/build_gcsfuse/main.go +++ b/tools/build_gcsfuse/main.go @@ -158,6 +158,7 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro fmt.Sprintf("GOROOT=%s", runtime.GOROOT()), fmt.Sprintf("GOPATH=%s", gopath), fmt.Sprintf("GOCACHE=%s", gocache), + "CGO_ENABLED=0", } // Build. diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index c9ff6cb244..18184c7dc0 100644 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -108,7 +108,7 @@ git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt #run tests with testbucket flag set +e -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt +GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt if [ $? -ne 0 ]; then diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index a36d394c97..b638978d4b 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -34,7 +34,7 @@ ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.20.4 as gcsfuse-package +FROM golang:1.20.5 as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler diff --git a/tools/integration_tests/run_tests_mounted_directory.sh b/tools/integration_tests/run_tests_mounted_directory.sh index 9e07509562..5cabf79eee 100644 --- a/tools/integration_tests/run_tests_mounted_directory.sh +++ b/tools/integration_tests/run_tests_mounted_directory.sh @@ -21,6 +21,7 @@ TEST_BUCKET_NAME=$1 MOUNT_DIR=$2 +export CGO_ENABLED=0 # Run integration tests for operations directory with static mounting gcsfuse --enable-storage-client-library=true --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 48abaaa077..623984b49a 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -17,7 +17,7 @@ # Copy the gcsfuse packages to the host: # > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output -FROM golang:1.20.4 as builder +FROM golang:1.20.5 as builder RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler