From 615444dcbd0d78b89ea631a48401c486629280ea Mon Sep 17 00:00:00 2001 From: Jonathan Hult Date: Fri, 9 Jul 2021 09:08:22 -0500 Subject: [PATCH 1/5] Fix #16387 - rootless Docker user (#16388) Move comment to top of USER instruction --- Dockerfile.rootless | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.rootless b/Dockerfile.rootless index efdb3bb72889..d3ba15f109da 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -56,7 +56,8 @@ COPY docker/rootless / COPY --from=build-env --chown=root:root /go/src/code.gitea.io/gitea/gitea /usr/local/bin/gitea COPY --from=build-env --chown=root:root /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini -USER 1000:1000 # git:git +#git:git +USER 1000:1000 ENV GITEA_WORK_DIR /var/lib/gitea ENV GITEA_CUSTOM /var/lib/gitea/custom ENV GITEA_TEMP /tmp/gitea From 8cbb38f546be78cedf101dfe2d00210738eb56f0 Mon Sep 17 00:00:00 2001 From: Avahe Kellenberger Date: Fri, 9 Jul 2021 20:16:07 -0400 Subject: [PATCH 2/5] Added documentation about 413 errors with an nginx solution (#15313) * Added documentation about 413 errors with an nginx solution. Co-authored-by: silverwind Co-authored-by: Norwin Co-authored-by: techknowlogick --- docs/content/doc/help/faq.en-us.md | 6 ++++++ docs/content/doc/usage/reverse-proxies.en-us.md | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/content/doc/help/faq.en-us.md b/docs/content/doc/help/faq.en-us.md index ff233d31e36c..953aa9f0120d 100644 --- a/docs/content/doc/help/faq.en-us.md +++ b/docs/content/doc/help/faq.en-us.md @@ -85,6 +85,12 @@ If certain clone options aren't showing up (HTTP/S or SSH), the following option `DISABLE_SSH`: if set to true, there will be no SSH link `SSH_EXPOSE_ANONYMOUS`: if set to false, SSH links will be hidden for anonymous users +## File upload fails with: 413 Request Entity Too Large + +This error occurs when the reverse proxy limits the file upload size. + +See the [reverse proxy guide]({{< relref "doc/usage/reverse-proxies.en-us.md" >}}) for a solution with nginx. + ## Custom Templates not loading or working incorrectly Gitea's custom templates must be added to the correct location or Gitea will not find and use them. diff --git a/docs/content/doc/usage/reverse-proxies.en-us.md b/docs/content/doc/usage/reverse-proxies.en-us.md index b339048d6843..5f1e6685ecfb 100644 --- a/docs/content/doc/usage/reverse-proxies.en-us.md +++ b/docs/content/doc/usage/reverse-proxies.en-us.md @@ -120,6 +120,14 @@ server { } ``` +## Resolving Error: 413 Request Entity Too Large + +This error indicates nginx is configured to restrict the file upload size. + +In your nginx config file containing your Gitea proxy directive, find the `location { ... }` block for Gitea and add the line +`client_max_body_size 16M;` to set this limit to 16 megabytes or any other number of choice. + + ## Apache HTTPD If you want Apache HTTPD to serve your Gitea instance, you can add the following to your Apache HTTPD configuration (usually located at `/etc/apache2/httpd.conf` in Ubuntu): From 7a402067ce086fea94685df0ae7db594d9a0f179 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Sat, 10 Jul 2021 00:24:52 +0000 Subject: [PATCH 3/5] [skip ci] Updated translations via Crowdin --- options/locale/locale_de-DE.ini | 3 +++ options/locale/locale_pt-PT.ini | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 0320102724eb..74fe7ebe9d91 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -100,6 +100,8 @@ never=Niemals [error] occurred=Ein Fehler ist aufgetreten report_message=Wenn du dir sicher bist, dass dies ein Gitea-Fehler ist, suche bitte auf GitHub nach diesem Fehler und erstelle gegebenenfalls einen neuen Bugreport. +missing_csrf=Fehlerhafte Anfrage: Kein CSRF Token verfügbar +invalid_csrf=Fehlerhafte Anfrage: Ungültiger CSRF Token [startpage] app_desc=Ein einfacher, selbst gehosteter Git-Service @@ -1922,6 +1924,7 @@ settings.archive.success=Das Repo wurde erfolgreich archiviert. settings.archive.error=Beim Versuch, das Repository zu archivieren, ist ein Fehler aufgetreten. Weitere Details finden sich im Log. settings.archive.error_ismirror=Du kannst keinen Repo-Mirror archivieren. settings.archive.branchsettings_unavailable=Branch-Einstellungen sind nicht verfügbar wenn das Repo archiviert ist. +settings.archive.tagsettings_unavailable=Tag Einstellungen sind nicht verfügbar, wenn das Repo archiviert wurde. settings.unarchive.button=Archivieren rückgängig machen settings.unarchive.header=Archivieren dieses Repos rückgängig machen settings.unarchive.text=Durch das Aufheben der Archivierung kann das Repo wieder Commits und Pushes sowie neue Issues und Pull-Requests empfangen. diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 7183f91e0082..ec2ec1f4b18d 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -1182,8 +1182,8 @@ issues.opened_by_fake=de %[2]s aberto %[1]s issues.closed_by_fake=de %[2]s fechada %[1]s issues.previous=Anterior issues.next=Seguinte -issues.open_title=Abertas -issues.closed_title=Fechadas +issues.open_title=Aberta +issues.closed_title=Fechada issues.num_comments=%d comentários issues.commented_at=`comentado %s` issues.delete_comment_confirm=Tem a certeza que quer eliminar este comentário? @@ -1414,11 +1414,11 @@ pulls.waiting_count_n=%d revisões pendentes pulls.wrong_commit_id=ID do cometimento tem que ser um ID de cometimento no ramo de destino pulls.no_merge_desc=A integração constante neste pedido não pode ser executada porque todas as opções de integração do repositório estão desabilitadas. -pulls.no_merge_helper=Habilite as opções de integração nas configurações do repositório ou faça manualmente a integração constante no pedido. +pulls.no_merge_helper=Habilite as opções de integração nas configurações do repositório ou faça a integração manualmente. pulls.no_merge_wip=A integração constante neste pedido não pode ser executada porque está marcada como sendo trabalho em andamento. pulls.no_merge_not_ready=A integração constante neste pedido não pode ser executada. Verifique o estado da revisão e as verificações de estado. pulls.no_merge_access=Não tem autorização para executar a integração constante neste pedido. -pulls.merge_pull_request=Executar a integração constante no pedido +pulls.merge_pull_request=Executar a integração pulls.rebase_merge_pull_request=Mudar a base e integrar pulls.rebase_merge_commit_pull_request=Mudar a base e integrar (--no-ff) pulls.squash_merge_pull_request=Comprimir e integrar From 07284792d40c556ec4c9e1c92379a922e746e12b Mon Sep 17 00:00:00 2001 From: Meano Date: Sun, 11 Jul 2021 01:40:14 +0800 Subject: [PATCH 4/5] Fix invalid params and typo of email templates (#16394) Signed-off-by: Meano --- options/locale/locale_en-US.ini | 4 ++-- templates/mail/auth/activate.tmpl | 6 +++--- templates/mail/auth/activate_email.tmpl | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 7f58a11e7385..191cb5de6764 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -326,8 +326,8 @@ hi_user_x = Hi %s, activate_account = Please activate your account activate_account.title = %s, please activate your account -activate_account.test_1 = Hi %[1]s, thanks for registering at %[2]s! -activate_account.test_2 = Please click the following link to activate your account within %s: +activate_account.text_1 = Hi %[1]s, thanks for registering at %[2]s! +activate_account.text_2 = Please click the following link to activate your account within %s: activate_email = Verify your email address activate_email.title = %s, please verify your e-mail address diff --git a/templates/mail/auth/activate.tmpl b/templates/mail/auth/activate.tmpl index 1f0204157895..ad34d9eebe9e 100644 --- a/templates/mail/auth/activate.tmpl +++ b/templates/mail/auth/activate.tmpl @@ -7,9 +7,9 @@ {{ $activate_url := printf "%suser/activate?code=%s" AppUrl .Code}} -

{{.i18n.Tr "mail.activate_account.test_1" .DisplayName AppName | Str2html}}


-

{{.i18n.Tr "mail.activate_account.test_2" .ActiveCodeLives | Str2html}}

{{$activate_url}}


-

{{.i18n.Tr "mail.link_not_working_do_paste" .DisplayName AppName | Str2html}}

+

{{.i18n.Tr "mail.activate_account.text_1" .DisplayName AppName | Str2html}}


+

{{.i18n.Tr "mail.activate_account.text_2" .ActiveCodeLives | Str2html}}

{{$activate_url}}


+

{{.i18n.Tr "mail.link_not_working_do_paste"}}

© {{AppName}}

diff --git a/templates/mail/auth/activate_email.tmpl b/templates/mail/auth/activate_email.tmpl index 27cff3ba4c51..6a8de5011207 100644 --- a/templates/mail/auth/activate_email.tmpl +++ b/templates/mail/auth/activate_email.tmpl @@ -9,7 +9,7 @@

{{.i18n.Tr "mail.hi_user_x" .DisplayName | Str2html}}


{{.i18n.Tr "mail.activate_email.text" .ActiveCodeLives | Str2html}}

{{$activate_url}}


-

{{.i18n.Tr "mail.link_not_working_do_paste" .DisplayName AppName | Str2html}}

+

{{.i18n.Tr "mail.link_not_working_do_paste"}}

© {{AppName}}

From 2f725cbc9e836e1d11b8645ee11ed2c82da8c0b7 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 10 Jul 2021 22:54:15 +0100 Subject: [PATCH 5/5] Add LRU mem cache implementation (#16226) The current default memory cache implementation is unbounded in size and number of objects cached. This is hardly ideal. This PR proposes creating a TwoQueue LRU cache as the underlying cache for Gitea. The cache is limited by the number of objects stored in the cache (rather than size) for simplicity. The default number of objects is 50000 - which is perhaps too small as most of our objects cached are going to be much less than 1kB. It may be worth considering using a different LRU implementation that actively limits sizes or avoids GC - however, this is just a beginning implementation. Signed-off-by: Andrew Thornton --- custom/conf/app.example.ini | 3 +- .../doc/advanced/config-cheat-sheet.en-us.md | 7 +- modules/cache/cache_twoqueue.go | 204 ++++++++++++++++++ modules/setting/cache.go | 7 +- routers/init.go | 4 +- 5 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 modules/cache/cache_twoqueue.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 22244545f589..251ec7a80e1c 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1471,7 +1471,7 @@ PATH = ;; if the cache enabled ;ENABLED = true ;; -;; Either "memory", "redis", or "memcache", default is "memory" +;; Either "memory", "redis", "memcache", or "twoqueue". default is "memory" ;ADAPTER = memory ;; ;; For "memory" only, GC interval in seconds, default is 60 @@ -1480,6 +1480,7 @@ PATH = ;; For "redis" and "memcache", connection host address ;; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180 ;; memcache: `127.0.0.1:11211` +;; twoqueue: `{"size":50000,"recent_ratio":0.25,"ghost_ratio":0.5}` or `50000` ;HOST = ;; ;; Time to keep items in cache if not used, default is 16 hours. diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 741c5f292cdf..611a7a887a9f 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -590,11 +590,12 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type ## Cache (`cache`) - `ENABLED`: **true**: Enable the cache. -- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`. -- `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only. -- `HOST`: **\**: Connection string for `redis` and `memcache`. +- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, `twoqueue` or `memcache`. (`twoqueue` represents a size limited LRU cache.) +- `INTERVAL`: **60**: Garbage Collection interval (sec), for memory and twoqueue cache only. +- `HOST`: **\**: Connection string for `redis` and `memcache`. For `twoqueue` sets configuration for the queue. - Redis: `redis://:macaron@127.0.0.1:6379/0?pool_size=100&idle_timeout=180s` - Memcache: `127.0.0.1:9090;127.0.0.1:9091` + - TwoQueue LRU cache: `{"size":50000,"recent_ratio":0.25,"ghost_ratio":0.5}` or `50000` representing the maximum number of objects stored in the cache. - `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching. ## Cache - LastCommitCache settings (`cache.last_commit`) diff --git a/modules/cache/cache_twoqueue.go b/modules/cache/cache_twoqueue.go new file mode 100644 index 000000000000..7d8fa7c93416 --- /dev/null +++ b/modules/cache/cache_twoqueue.go @@ -0,0 +1,204 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "strconv" + "sync" + "time" + + mc "gitea.com/go-chi/cache" + lru "github.com/hashicorp/golang-lru" + jsoniter "github.com/json-iterator/go" +) + +// TwoQueueCache represents a LRU 2Q cache adapter implementation +type TwoQueueCache struct { + lock sync.Mutex + cache *lru.TwoQueueCache + interval int +} + +// TwoQueueCacheConfig describes the configuration for TwoQueueCache +type TwoQueueCacheConfig struct { + Size int `ini:"SIZE" json:"size"` + RecentRatio float64 `ini:"RECENT_RATIO" json:"recent_ratio"` + GhostRatio float64 `ini:"GHOST_RATIO" json:"ghost_ratio"` +} + +// MemoryItem represents a memory cache item. +type MemoryItem struct { + Val interface{} + Created int64 + Timeout int64 +} + +func (item *MemoryItem) hasExpired() bool { + return item.Timeout > 0 && + (time.Now().Unix()-item.Created) >= item.Timeout +} + +var _ mc.Cache = &TwoQueueCache{} + +// Put puts value into cache with key and expire time. +func (c *TwoQueueCache) Put(key string, val interface{}, timeout int64) error { + item := &MemoryItem{ + Val: val, + Created: time.Now().Unix(), + Timeout: timeout, + } + c.lock.Lock() + defer c.lock.Unlock() + c.cache.Add(key, item) + return nil +} + +// Get gets cached value by given key. +func (c *TwoQueueCache) Get(key string) interface{} { + c.lock.Lock() + defer c.lock.Unlock() + cached, ok := c.cache.Get(key) + if !ok { + return nil + } + item, ok := cached.(*MemoryItem) + + if !ok || item.hasExpired() { + c.cache.Remove(key) + return nil + } + + return item.Val +} + +// Delete deletes cached value by given key. +func (c *TwoQueueCache) Delete(key string) error { + c.lock.Lock() + defer c.lock.Unlock() + c.cache.Remove(key) + return nil +} + +// Incr increases cached int-type value by given key as a counter. +func (c *TwoQueueCache) Incr(key string) error { + c.lock.Lock() + defer c.lock.Unlock() + cached, ok := c.cache.Get(key) + if !ok { + return nil + } + item, ok := cached.(*MemoryItem) + + if !ok || item.hasExpired() { + c.cache.Remove(key) + return nil + } + + var err error + item.Val, err = mc.Incr(item.Val) + return err +} + +// Decr decreases cached int-type value by given key as a counter. +func (c *TwoQueueCache) Decr(key string) error { + c.lock.Lock() + defer c.lock.Unlock() + cached, ok := c.cache.Get(key) + if !ok { + return nil + } + item, ok := cached.(*MemoryItem) + + if !ok || item.hasExpired() { + c.cache.Remove(key) + return nil + } + + var err error + item.Val, err = mc.Decr(item.Val) + return err +} + +// IsExist returns true if cached value exists. +func (c *TwoQueueCache) IsExist(key string) bool { + c.lock.Lock() + defer c.lock.Unlock() + cached, ok := c.cache.Peek(key) + if !ok { + return false + } + item, ok := cached.(*MemoryItem) + if !ok || item.hasExpired() { + c.cache.Remove(key) + return false + } + + return true +} + +// Flush deletes all cached data. +func (c *TwoQueueCache) Flush() error { + c.lock.Lock() + defer c.lock.Unlock() + c.cache.Purge() + return nil +} + +func (c *TwoQueueCache) checkAndInvalidate(key interface{}) { + c.lock.Lock() + defer c.lock.Unlock() + cached, ok := c.cache.Peek(key) + if !ok { + return + } + item, ok := cached.(*MemoryItem) + if !ok || item.hasExpired() { + c.cache.Remove(item) + } +} + +func (c *TwoQueueCache) startGC() { + if c.interval < 0 { + return + } + for _, key := range c.cache.Keys() { + c.checkAndInvalidate(key) + } + time.AfterFunc(time.Duration(c.interval)*time.Second, c.startGC) +} + +// StartAndGC starts GC routine based on config string settings. +func (c *TwoQueueCache) StartAndGC(opts mc.Options) error { + var err error + size := 50000 + if opts.AdapterConfig != "" { + size, err = strconv.Atoi(opts.AdapterConfig) + } + if err != nil { + json := jsoniter.ConfigCompatibleWithStandardLibrary + if !json.Valid([]byte(opts.AdapterConfig)) { + return err + } + + cfg := &TwoQueueCacheConfig{ + Size: 50000, + RecentRatio: lru.Default2QRecentRatio, + GhostRatio: lru.Default2QGhostEntries, + } + _ = json.Unmarshal([]byte(opts.AdapterConfig), cfg) + c.cache, err = lru.New2QParams(cfg.Size, cfg.RecentRatio, cfg.GhostRatio) + } else { + c.cache, err = lru.New2Q(size) + } + c.interval = opts.Interval + if c.interval > 0 { + go c.startGC() + } + return err +} + +func init() { + mc.Register("twoqueue", &TwoQueueCache{}) +} diff --git a/modules/setting/cache.go b/modules/setting/cache.go index 7bfea919618e..2bfe2318f547 100644 --- a/modules/setting/cache.go +++ b/modules/setting/cache.go @@ -58,11 +58,16 @@ func newCacheService() { log.Fatal("Failed to map Cache settings: %v", err) } - CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}) + CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache", "twoqueue"}) switch CacheService.Adapter { case "memory": case "redis", "memcache": CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ") + case "twoqueue": + CacheService.Conn = strings.TrimSpace(sec.Key("HOST").String()) + if CacheService.Conn == "" { + CacheService.Conn = "50000" + } case "": // disable cache CacheService.Enabled = false default: diff --git a/routers/init.go b/routers/init.go index 05dbe4bd664e..3ee7c7357285 100644 --- a/routers/init.go +++ b/routers/init.go @@ -52,7 +52,9 @@ func NewServices() { log.Fatal("repository init failed: %v", err) } mailer.NewContext() - _ = cache.NewContext() + if err := cache.NewContext(); err != nil { + log.Fatal("Unable to start cache service: %v", err) + } notification.NewContext() if err := archiver.Init(); err != nil { log.Fatal("archiver init failed: %v", err)