[3.5] backport health check e2e tests.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

pkg/expect/expect.go

@@ -203,3 +203,7 @@ func (ep *ExpectProcess) Lines() []string {
 	defer ep.mu.Unlock()
 	return ep.lines
 }
+
+func (ep *ExpectProcess) IsRunning() bool {
+	return ep.cmd != nil
+}

server/etcdserver/server.go

@@ -2152,6 +2152,7 @@ func (s *EtcdServer) apply(
 			zap.Stringer("type", e.Type))
 		switch e.Type {
 		case raftpb.EntryNormal:
+			// gofail: var beforeApplyOneEntryNormal struct{}
 			s.applyEntryNormal(&e)
 			s.setAppliedIndex(e.Index)
 			s.setTerm(e.Term)

server/mvcc/backend/batch_tx.go

@@ -307,6 +307,7 @@ func newBatchTxBuffered(backend *backend) *batchTxBuffered {
 func (t *batchTxBuffered) Unlock() {
 	if t.pending != 0 {
 		t.backend.readTx.Lock() // blocks txReadBuffer for writing.
+		// gofail: var beforeWritebackBuf struct{}
 		t.buf.writeback(&t.backend.readTx.buf)
 		t.backend.readTx.Unlock()
 		if t.pending >= t.backend.batchLimit {

tests/e2e/etcdctl.go

@@ -55,6 +55,15 @@ func (ctl *Etcdctl) Put(key, value string) error {
 	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "OK")
 }
 
+func (ctl *Etcdctl) PutWithAuth(key, value, username, password string) error {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	args := ctl.cmdArgs()
+	args = append(args, "--user", fmt.Sprintf("%s:%s", username, password), "put", key, value)
+	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "OK")
+}
+
 func (ctl *Etcdctl) Set(key, value string) error {
 	if !ctl.v2 {
 		panic("Unsupported method for v3")
@@ -72,6 +81,32 @@ func (ctl *Etcdctl) Set(key, value string) error {
 	return nil
 }
 
+func (ctl *Etcdctl) AuthEnable() error {
+	args := ctl.cmdArgs("auth", "enable")
+	return e2e.SpawnWithExpectWithEnv(args, ctl.env(), "Authentication Enabled")
+}
+
+func (ctl *Etcdctl) UserGrantRole(user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) {
+	var resp clientv3.AuthUserGrantRoleResponse
+	err := ctl.spawnJsonCmd(&resp, "user", "grant-role", user, role)
+	return &resp, err
+}
+
+func (ctl *Etcdctl) UserAdd(name, password string) (*clientv3.AuthUserAddResponse, error) {
+	args := []string{"user", "add"}
+	if password == "" {
+		args = append(args, name)
+		args = append(args, "--no-password")
+	} else {
+		args = append(args, fmt.Sprintf("%s:%s", name, password))
+	}
+	args = append(args, "--interactive=false")
+
+	var resp clientv3.AuthUserAddResponse
+	err := ctl.spawnJsonCmd(&resp, args...)
+	return &resp, err
+}
+
 func (ctl *Etcdctl) AlarmList() (*clientv3.AlarmResponse, error) {
 	if ctl.v2 {
 		panic("Unsupported method for v2")