Merge branch 'main' into shared_txReadBuffer

etcd-io · May 18, 2021 · 88d26c1 · 88d26c1
2 parents acaf5ae + 46b49a6
commit 88d26c1
Show file tree

Hide file tree

Showing 223 changed files with 5,271 additions and 2,279 deletions.
diff --git a/.github/ISSUE_TEMPLATE/distributors-application.md b/.github/ISSUE_TEMPLATE/distributors-application.md
@@ -20,9 +20,9 @@ meeting the membership criteria.
 **4. Be a participant and active contributor in the community.**
 
 **5. Accept the Embargo Policy.**
-<!-- https://github.com/etcd-io/etcd/blob/master/security/security-release-process.md#disclosures -->
+<!-- https://github.com/etcd-io/etcd/blob/main/security/security-release-process.md#disclosures -->
 
 **6. Be willing to contribute back.**
-<!-- Per https://github.com/etcd-io/etcd/blob/master/security/security-release-process.md#patch-release-and-public-communication -->
+<!-- Per https://github.com/etcd-io/etcd/blob/main/security/security-release-process.md#patch-release-and-public-communication -->
 
 **7. Have someone already on the list vouch for the person requesting membership on behalf of your distribution.**
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,2 +1,2 @@
 
-Please read https://github.com/etcd-io/etcd/blob/master/CONTRIBUTING.md#contribution-flow.
+Please read https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#contribution-flow.
diff --git a/.github/SECURITY.md b/.github/SECURITY.md
@@ -1,2 +1,2 @@
 
-Please read https://github.com/etcd-io/etcd/blob/master/security/README.md.
+Please read https://github.com/etcd-io/etcd/blob/main/security/README.md.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ master, release-0.4, release-2.0, release-2.1, release-2.2, release-2.3, release-3.0, release-3.1 ]
+    branches: [ main, release-0.4, release-2.0, release-2.1, release-2.2, release-2.3, release-3.0, release-3.1 ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [ main ]
   schedule:
     - cron: '20 14 * * 5'
 

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,35 @@
+name: E2E
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+        - linux-amd64-e2e
+        - linux-386-e2e
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: "^1.16"
+    - run: date
+    - env:
+        TARGET: ${{ matrix.target }}
+      run: |
+        echo "${TARGET}"
+        case "${TARGET}" in
+          linux-amd64-e2e)
+            PASSES='build release e2e' MANUAL_VER=v3.4.7 CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log
+            ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            ;;
+          linux-386-e2e)
+            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log
+            ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            ;;
+          *)
+            echo "Failed to find target"
+            exit 1
+            ;;
+        esac
diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml
@@ -0,0 +1,29 @@
+name: functional-tests
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+        - linux-amd64-functional
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: "^1.16"
+    - run: date
+    - env:
+        TARGET: ${{ matrix.target }}
+      run: |
+        echo "${TARGET}"
+        case "${TARGET}" in
+          linux-amd64-functional)
+            GO_BUILD_FLAGS='-v -mod=readonly' ./build && GOARCH=amd64 PASSES='functional' ./test
+            ;;
+          *)
+            echo "Failed to find target"
+            exit 1
+            ;;
+        esac
diff --git a/.github/workflows/grpcproxy.yaml b/.github/workflows/grpcproxy.yaml
@@ -0,0 +1,30 @@
+name: grpcProxy-tests
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+        - linux-amd64-grpcproxy
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: "^1.16"
+    - run: date
+    - env:
+        TARGET: ${{ matrix.target }}
+      run: |
+        echo "${TARGET}"
+        case "${TARGET}" in
+          linux-amd64-grpcproxy)
+            PASSES='build grpcproxy'  CPU='4' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log
+            ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            ;;
+          *)
+            echo "Failed to find target"
+            exit 1
+            ;;
+        esac
diff --git a/.github/workflows/self-hosted-linux-arm64-graviton2-tests.yml b/.github/workflows/self-hosted-linux-arm64-graviton2-tests.yml
@@ -0,0 +1,19 @@
+name: Linux ARM64 Graviton2
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: [self-hosted, linux, ARM64, graviton2]
+    steps:
+      - uses: actions/checkout@v2
+      - name: install dependencies
+        run: |
+          sudo amazon-linux-extras install epel
+          sudo yum install -y git gcc ShellCheck
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "^1.16"
+      - run: go version
+      - run: date
+      - name: Run tests
+        run: TEST_OPTS="PASSES='fmt bom dep build unit integration_e2e'" make test
diff --git a/.github/workflows/self-hosted-linux-arm64-graviton2-tests.yml.md b/.github/workflows/self-hosted-linux-arm64-graviton2-tests.yml.md
@@ -0,0 +1,142 @@
+
+## Graviton-based self-hosted github action worker
+
+### Step 1. Create an EC2 instance with Graviton
+
+Create an AWS Graviton-based EC2 instance. For example,
+
+```
+# or download from https://github.com/aws/aws-k8s-tester/releases
+cd ${HOME}/go/src/github.com/aws/aws-k8s-tester
+go install -v ./cmd/ec2-utils
+
+# create arm64 AL2 instance
+AWS_K8S_TESTER_EC2_ON_FAILURE_DELETE=true \
+AWS_K8S_TESTER_EC2_LOG_COLOR=true \
+AWS_K8S_TESTER_EC2_REGION=us-west-2 \
+AWS_K8S_TESTER_EC2_S3_BUCKET_CREATE=true \
+AWS_K8S_TESTER_EC2_S3_BUCKET_CREATE_KEEP=true \
+AWS_K8S_TESTER_EC2_REMOTE_ACCESS_KEY_CREATE=true \
+AWS_K8S_TESTER_EC2_ASGS_FETCH_LOGS=false \
+AWS_K8S_TESTER_EC2_ASGS='{"GetRef.Name-arm64-al2-cpu":{"name":"GetRef.Name-arm64-al2-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_arm_64","image-id-ssm-parameter":"/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-arm64-gp2","instance-types":["m6g.xlarge"],"volume-size":40,"asg-min-size":1,"asg-max-size":1,"asg-desired-capacity":1}}' \
+AWS_K8S_TESTER_EC2_ROLE_CREATE=true \
+AWS_K8S_TESTER_EC2_VPC_CREATE=true \
+ec2-utils create instances --enable-prompt=true --auto-path
+```
+
+### Step 2. Install github action on the host
+
+SSH into the instance, and install the github action self-hosted runner (see [install scripts](https://github.com/etcd-io/etcd/settings/actions/runners/new?arch=arm64&os=linux)).
+
+### Step 3. Configure github action on the host
+
+SSH into the instance, and configure the github action self-hosted runner.
+
+First, we need disable ICU install (see [actions/runner issue on ARM64](https://github.com/actions/runner/issues/629)):
+
+```
+sudo yum install -y patch
+```
+
+And write this bash script:
+
+```
+#!/bin/bash -e
+
+patch -p1 <<ICU_PATCH
+diff -Naur a/bin/Runner.Listener.runtimeconfig.json b/bin/Runner.Listener.runtimeconfig.json
+--- a/bin/Runner.Listener.runtimeconfig.json	2020-07-01 02:21:09.000000000 +0000
++++ b/bin/Runner.Listener.runtimeconfig.json	2020-07-28 00:02:38.748868613 +0000
+@@ -8,7 +8,8 @@
+       }
+     ],
+     "configProperties": {
+-      "System.Runtime.TieredCompilation.QuickJit": true
++      "System.Runtime.TieredCompilation.QuickJit": true,
++      "System.Globalization.Invariant": true
+     }
+   }
+-}
+\ No newline at end of file
++}
+diff -Naur a/bin/Runner.PluginHost.runtimeconfig.json b/bin/Runner.PluginHost.runtimeconfig.json
+--- a/bin/Runner.PluginHost.runtimeconfig.json	2020-07-01 02:21:22.000000000 +0000
++++ b/bin/Runner.PluginHost.runtimeconfig.json	2020-07-28 00:02:59.358680003 +0000
+@@ -8,7 +8,8 @@
+       }
+     ],
+     "configProperties": {
+-      "System.Runtime.TieredCompilation.QuickJit": true
++      "System.Runtime.TieredCompilation.QuickJit": true,
++      "System.Globalization.Invariant": true
+     }
+   }
+-}
+\ No newline at end of file
++}
+diff -Naur a/bin/Runner.Worker.runtimeconfig.json b/bin/Runner.Worker.runtimeconfig.json
+--- a/bin/Runner.Worker.runtimeconfig.json	2020-07-01 02:21:16.000000000 +0000
++++ b/bin/Runner.Worker.runtimeconfig.json	2020-07-28 00:02:19.159028531 +0000
+@@ -8,7 +8,8 @@
+       }
+     ],
+     "configProperties": {
+-      "System.Runtime.TieredCompilation.QuickJit": true
++      "System.Runtime.TieredCompilation.QuickJit": true,
++      "System.Globalization.Invariant": true
+     }
+   }
+-}
+\ No newline at end of file
++}
+ICU_PATCH
+```
+
+And now patch the github action runner:
+
+```
+cd ${HOME}/actions-runner
+bash patch.sh
+```
+
+```
+patching file bin/Runner.Listener.runtimeconfig.json
+patching file bin/Runner.PluginHost.runtimeconfig.json
+patching file bin/Runner.Worker.runtimeconfig.json
+```
+
+And now configure:
+
+```
+sudo yum install -y wget
+INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
+echo ${INSTANCE_ID}
+
+# get token from https://github.com/etcd-io/etcd/settings/actions/runners/new?arch=arm64&os=linux
+cd ${HOME}/actions-runner
+./config.sh \
+--work "_work" \
+--name ${INSTANCE_ID} \
+--labels self-hosted,linux,ARM64,graviton2 \
+--url https://github.com/etcd-io/etcd \
+--token ...
+```
+
+And run:
+
+```
+# run this as a process in the terminal
+cd ${HOME}/actions-runner
+./run.sh
+
+# or run this as a systemd service
+cd ${HOME}/actions-runner
+sudo ./svc.sh install
+sudo ./svc.sh start
+sudo ./svc.sh status
+```
+
+### Step 4. Create github action configuration
+
+See https://github.com/etcd-io/etcd/pull/12928.
+
diff --git a/.github/workflows/unit-tests.yaml → .github/workflows/tests.yaml b/.github/workflows/unit-tests.yaml → .github/workflows/tests.yaml
@@ -1,32 +1,24 @@
-name: Unit Tests
-on: [push]
+name: Tests
+on: [push, pull_request]
 jobs:
   test:
-    # TODO: Remove continue-on-error after migrating from Travis to GitHub Actions
-    continue-on-error: true
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        # TODO: Add tip when https://github.com/actions/setup-go/issues/21 is resolved
-        go: [1.16.3]
         target:
         - linux-amd64-fmt
-        - linux-amd64-grpcproxy
-        - linux-amd64-coverage
         - linux-amd64-integration-1-cpu
         - linux-amd64-integration-2-cpu
         - linux-amd64-integration-4-cpu
-        - linux-amd64-functional
         - linux-amd64-unit-4-cpu-race
         - all-build
-        - linux-amd64-fmt-unit-go-tip-2-cpu
         - linux-386-unit-1-cpu
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go }}
+        go-version: "^1.16"
     - run: date
     - env:
         TARGET: ${{ matrix.target }}
@@ -45,9 +37,6 @@ jobs:
           linux-amd64-integration-4-cpu)
             GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test.sh
             ;;
-          linux-amd64-functional)
-            GO_BUILD_FLAGS='-v -mod=readonly' ./build && GOARCH=amd64 PASSES='functional' ./test
-            ;;
           linux-amd64-unit-4-cpu-race)
             GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./test.sh -p=2
             ;;
@@ -61,18 +50,11 @@ jobs:
             GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=ppc64le ./build.sh
             GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build.sh
             ;;
-          linux-amd64-grpcproxy)
-            PASSES='build grpcproxy'  CPU='4' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log
-            ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
-            ;;
-          linux-amd64-coverage)
-            ./scripts/codecov_upload.sh test-coverage.log \
-            ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-coverage.log
-            ;;
-          linux-amd64-fmt-unit-go-tip-2-cpu)
-            GOARCH=amd64 PASSES='fmt unit' CPU='2' RACE='false' ./test.sh -p=2
-            ;;
           linux-386-unit-1-cpu)
             GOARCH=386 PASSES='unit' RACE='false' CPU='1' ./test -p=4
             ;;
+          *)
+            echo "Failed to find target"
+            exit 1
+            ;;
         esac