Add Graceful shutdown for Windows and hooks for shutdown of goroutines (

#8964)

* Graceful Shutdown for windows and others

Restructures modules/graceful, adding shutdown for windows, removing and
replacing the old minwinsvc code.

Creates a new waitGroup - terminate which allows for goroutines to
finish up after the shutdown of the servers.

Shutdown and terminate hooks are added for goroutines.

* Remove unused functions - these can be added in a different PR

* Add startup timeout functionality

* Document STARTUP_TIMEOUT

cmd/web.go

@@ -227,7 +227,8 @@ func runWeb(ctx *cli.Context) error {
 		log.Critical("Failed to start server: %v", err)
 	}
 	log.Info("HTTP Listener: %s Closed", listenAddr)
-	graceful.WaitForServers()
+	graceful.Manager.WaitForServers()
+	graceful.Manager.WaitForTerminate()
 	log.Close()
 	return nil
 }

cmd/web_graceful.go

@@ -1,5 +1,3 @@
-// +build !windows
-
 // Copyright 2016 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
@@ -27,11 +25,11 @@ func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Hand
 
 // NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
 func NoHTTPRedirector() {
-	graceful.InformCleanup()
+	graceful.Manager.InformCleanup()
 }
 
 // NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
 // for our main HTTP/HTTPS service
 func NoMainListener() {
-	graceful.InformCleanup()
+	graceful.Manager.InformCleanup()
 }

custom/conf/app.ini.sample

@@ -287,6 +287,9 @@ ALLOW_GRACEFUL_RESTARTS = true
 ; shutting down. Force shutdown if this process takes longer than this delay.
 ; set to a negative value to disable
 GRACEFUL_HAMMER_TIME = 60s
+; Allows the setting of a startup timeout and waithint for Windows as SVC service
+; 0 disables this.
+STARTUP_TIMEOUT = 0
 ; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
 STATIC_CACHE_TIME = 6h
 
@@ -897,4 +900,4 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
 ; Max attempts per http/https request on migrations.
 MAX_ATTEMPTS = 3
 ; Backoff time per http/https request retry (seconds)
-RETRY_BACKOFF = 3
+RETRY_BACKOFF = 3

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -189,6 +189,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LETSENCRYPT_EMAIL`: **email@example.com**: Email used by Letsencrypt to notify about problems with issued certificates. (No default)
 - `ALLOW_GRACEFUL_RESTARTS`: **true**: Perform a graceful restart on SIGHUP
 - `GRACEFUL_HAMMER_TIME`: **60s**: After a restart the parent process will stop accepting new connections and will allow requests to finish before stopping. Shutdown will be forced if it takes longer than this time.
+- `STARTUP_TIMEOUT`: **0**: Shutsdown the server if startup takes longer than the provided time. On Windows setting this sends a waithint to the SVC host to tell the SVC host startup may take some time. Please note startup is determined by the opening of the listeners - HTTP/HTTPS/SSH. Indexers may take longer to startup and can have their own timeouts.
 
 ## Database (`database`)
 

models/repo_indexer.go

@@ -84,7 +84,7 @@ func InitRepoIndexer() {
 	if setting.Indexer.StartupTimeout > 0 {
 		go func() {
 			timeout := setting.Indexer.StartupTimeout
-			if graceful.IsChild && setting.GracefulHammerTime > 0 {
+			if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
 				timeout += setting.GracefulHammerTime
 			}
 			select {

modules/graceful/manager.go

@@ -0,0 +1,187 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+type state uint8
+
+const (
+	stateInit state = iota
+	stateRunning
+	stateShuttingDown
+	stateTerminate
+)
+
+// There are three places that could inherit sockets:
+//
+// * HTTP or HTTPS main listener
+// * HTTP redirection fallback
+// * SSH
+//
+// If you add an additional place you must increment this number
+// and add a function to call manager.InformCleanup if it's not going to be used
+const numberOfServersToCreate = 3
+
+// Manager represents the graceful server manager interface
+var Manager *gracefulManager
+
+func init() {
+	Manager = newGracefulManager()
+}
+
+func (g *gracefulManager) doShutdown() {
+	if !g.setStateTransition(stateRunning, stateShuttingDown) {
+		return
+	}
+	g.lock.Lock()
+	close(g.shutdown)
+	g.lock.Unlock()
+
+	if setting.GracefulHammerTime >= 0 {
+		go g.doHammerTime(setting.GracefulHammerTime)
+	}
+	go func() {
+		g.WaitForServers()
+		<-time.After(1 * time.Second)
+		g.doTerminate()
+	}()
+}
+
+func (g *gracefulManager) doHammerTime(d time.Duration) {
+	time.Sleep(d)
+	select {
+	case <-g.hammer:
+	default:
+		log.Warn("Setting Hammer condition")
+		close(g.hammer)
+	}
+
+}
+
+func (g *gracefulManager) doTerminate() {
+	if !g.setStateTransition(stateShuttingDown, stateTerminate) {
+		return
+	}
+	g.lock.Lock()
+	close(g.terminate)
+	g.lock.Unlock()
+}
+
+// IsChild returns if the current process is a child of previous Gitea process
+func (g *gracefulManager) IsChild() bool {
+	return g.isChild
+}
+
+// IsShutdown returns a channel which will be closed at shutdown.
+// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
+func (g *gracefulManager) IsShutdown() <-chan struct{} {
+	g.lock.RLock()
+	if g.shutdown == nil {
+		g.lock.RUnlock()
+		g.lock.Lock()
+		if g.shutdown == nil {
+			g.shutdown = make(chan struct{})
+		}
+		defer g.lock.Unlock()
+		return g.shutdown
+	}
+	defer g.lock.RUnlock()
+	return g.shutdown
+}
+
+// IsHammer returns a channel which will be closed at hammer
+// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
+// Servers running within the running server wait group should respond to IsHammer
+// if not shutdown already
+func (g *gracefulManager) IsHammer() <-chan struct{} {
+	g.lock.RLock()
+	if g.hammer == nil {
+		g.lock.RUnlock()
+		g.lock.Lock()
+		if g.hammer == nil {
+			g.hammer = make(chan struct{})
+		}
+		defer g.lock.Unlock()
+		return g.hammer
+	}
+	defer g.lock.RUnlock()
+	return g.hammer
+}
+
+// IsTerminate returns a channel which will be closed at terminate
+// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
+// IsTerminate will only close once all running servers have stopped
+func (g *gracefulManager) IsTerminate() <-chan struct{} {
+	g.lock.RLock()
+	if g.terminate == nil {
+		g.lock.RUnlock()
+		g.lock.Lock()
+		if g.terminate == nil {
+			g.terminate = make(chan struct{})
+		}
+		defer g.lock.Unlock()
+		return g.terminate
+	}
+	defer g.lock.RUnlock()
+	return g.terminate
+}
+
+// ServerDone declares a running server done and subtracts one from the
+// running server wait group. Users probably do not want to call this
+// and should use one of the RunWithShutdown* functions
+func (g *gracefulManager) ServerDone() {
+	g.runningServerWaitGroup.Done()
+}
+
+// WaitForServers waits for all running servers to finish. Users should probably
+// instead use AtTerminate or IsTerminate
+func (g *gracefulManager) WaitForServers() {
+	g.runningServerWaitGroup.Wait()
+}
+
+// WaitForTerminate waits for all terminating actions to finish.
+// Only the main go-routine should use this
+func (g *gracefulManager) WaitForTerminate() {
+	g.terminateWaitGroup.Wait()
+}
+
+func (g *gracefulManager) getState() state {
+	g.lock.RLock()
+	defer g.lock.RUnlock()
+	return g.state
+}
+
+func (g *gracefulManager) setStateTransition(old, new state) bool {
+	if old != g.getState() {
+		return false
+	}
+	g.lock.Lock()
+	if g.state != old {
+		g.lock.Unlock()
+		return false
+	}
+	g.state = new
+	g.lock.Unlock()
+	return true
+}
+
+func (g *gracefulManager) setState(st state) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.state = st
+}
+
+// InformCleanup tells the cleanup wait group that we have either taken a listener
+// or will not be taking a listener
+func (g *gracefulManager) InformCleanup() {
+	g.createServerWaitGroup.Done()
+}