Skip to content

Commit

Permalink
fix pipeline stuck in queue status because etcd server abnormal
Browse files Browse the repository at this point in the history
  • Loading branch information
chengjoey committed Aug 21, 2024
1 parent c315ced commit 7cecc31
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions internal/tools/pipeline/providers/leaderworker/impl_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ package leaderworker

import (
"context"
"fmt"
"time"

"github.com/pkg/errors"
clientv3 "go.etcd.io/etcd/client/v3"

"github.com/erda-project/erda/internal/tools/pipeline/providers/leaderworker/lwctx"
Expand Down Expand Up @@ -151,6 +153,10 @@ func (p *provider) workerIntervalCleanupOnDelete(ctx context.Context, ev Event)
if err == nil {
return
}
// actively panic exit
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
panic(fmt.Errorf("actively panic exit, maybe the etcd server has been shut down, err: %v", err))
}
p.Log.Errorf("failed to do worker interval cleanup on delete(auto retry), step: delete heartbeat key, workerID: %s, err: %v", ev.WorkerID, err)
time.Sleep(p.Cfg.Worker.RetryInterval)
}
Expand Down

0 comments on commit 7cecc31

Please sign in to comment.