From 67662bb735967d6ecc130b7b51a2974133776857 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 24 Feb 2025 15:02:29 -0800 Subject: [PATCH] Run gitea with memcached cache adapter By default gitea caches everything in memory using a Go hashmap. There is suspicion that the now many persistent AI web crawlers cause this hashmap to grow in ways that eventually cause the Go GC system to pause the world in noticeable ways when loading pages. Restarting the gitea services seems to temporarily reset things (as it should with an in memory map) until we cross some threshold and things become slow again. The good news is that gitea supports several backends (called adapters) for the cache. We elect to use memcached because it is relatively simple and has a FOSS license (unlike redis). The other alternative we could consider is twoqueue which also caches within memory in the Go runtime but allows for setting a size limit. I've gone with memcached because it doesn't rely on Golang GC, but twoqueue is likely simpler if we want to start there. Note we also bump the job timeout to 5400 seconds (90 minutes) from 4800 seconds (80 minutes) because a run on ovh-gra1 timed out while running testinfra test cases (the very end of the job). It is possible that using memcache is slightly slower than using in process memory caching, but the goal here isn't to make things faster it is to make things more consistent over time. As long as memcached performance is within the same ballpark and doesn't degrade over time this is acceptable. Change-Id: Ie9ca246a8321fe84d9a1582e35cd4c5459b48bee --- playbooks/roles/gitea/tasks/main.yaml | 8 +++++-- playbooks/roles/gitea/templates/app.ini.j2 | 4 ++++ .../gitea/templates/docker-compose.yaml.j2 | 13 ++++++++++++ testinfra/test_gitea.py | 21 ++++++++++++++++++- zuul.d/system-config-run.yaml | 2 +- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/gitea/tasks/main.yaml b/playbooks/roles/gitea/tasks/main.yaml index 8b46c6de46..3235d5b37d 100644 --- a/playbooks/roles/gitea/tasks/main.yaml +++ b/playbooks/roles/gitea/tasks/main.yaml @@ -34,8 +34,12 @@ - name: Install distro packages package: name: + # TODO(clarkb) does the install-docker role handle these two packages? - docker-compose - python3-requests + # Installed to make checking memcached stats easy in testing and for + # human led debugging. + - netcat-openbsd state: present - name: Install reverse proxy @@ -63,9 +67,9 @@ shell: cmd: docker-compose stop --timeout 60 chdir: /etc/gitea-docker/ - - name: Run docker-compose up mariadb gitea-web + - name: Run docker-compose up mariadb memcached gitea-web shell: - cmd: docker-compose up -d --timeout 60 mariadb gitea-web + cmd: docker-compose up -d --timeout 60 mariadb memcached gitea-web chdir: /etc/gitea-docker/ # We wait here for the main gitea service to start before starting diff --git a/playbooks/roles/gitea/templates/app.ini.j2 b/playbooks/roles/gitea/templates/app.ini.j2 index 013c3dfbc2..cb35ee6fee 100644 --- a/playbooks/roles/gitea/templates/app.ini.j2 +++ b/playbooks/roles/gitea/templates/app.ini.j2 @@ -30,6 +30,10 @@ PASSWD = {{ gitea_db_password }} SSL_MODE = disable LOG_SQL = false +[cache] +ADAPTER = memcache +HOST = 127.0.0.1:11211 + [repository] ROOT = /data/git/repositories DISABLED_REPO_UNITS = repo.issues,repo.pulls,repo.wiki,repo.projects,repo.actions diff --git a/playbooks/roles/gitea/templates/docker-compose.yaml.j2 b/playbooks/roles/gitea/templates/docker-compose.yaml.j2 index 5f0ca67c24..60185e5515 100644 --- a/playbooks/roles/gitea/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/gitea/templates/docker-compose.yaml.j2 @@ -20,9 +20,22 @@ services: driver: journald options: tag: "docker-mariadb" + memcached: + image: quay.io/opendevmirror/memcached:latest + network_mode: host + restart: always + command: + - -v + - --listen=127.0.0.1:11211 + - --memory-limit=1024 + logging: + driver: journald + options: + tag: "docker-memcached" gitea-web: depends_on: - mariadb + - memcached image: docker.io/opendevorg/gitea:latest network_mode: host restart: always diff --git a/testinfra/test_gitea.py b/testinfra/test_gitea.py index 42a6a6e7aa..4af24bbc16 100644 --- a/testinfra/test_gitea.py +++ b/testinfra/test_gitea.py @@ -26,6 +26,8 @@ def test_gitea_listening(host): assert gitea_ssh.is_listening gitea_proxy = host.socket("tcp://0.0.0.0:3081") assert gitea_proxy.is_listening + memcached = host.socket("tcp://127.0.0.1:11211") + assert memcached.is_listening def test_ulimit(host): cmd = host.run("/usr/local/bin/docker-compose " @@ -73,6 +75,12 @@ def test_ondisk_logs(host): mariadb_log = host.file('/var/log/containers/docker-mariadb.log') assert mariadb_log.exists + # Commented out for now as memcached logging is either very quiet and + # we don't create the file or very verbose and far too chatty for + # production (basically no logs or every set and get is logged). + #memcached_log = host.file('/var/log/containers/docker-memcached.log') + #assert memcached_log.exists + gitea_log = host.file('/var/log/containers/docker-gitea.log') assert gitea_log.exists @@ -135,11 +143,14 @@ def test_no_500_template_content(host): assert 'Internal Server Error' not in cmd.stdout def test_gitea_screenshots(host): - shots = ( ('https://localhost:3081', None, 'gitea-main.png'), ('https://localhost:3081/opendev/system-config', None, 'gitea-project-system-config.png'), + # Fetch system-config twice to ensure we exercise the caching + # system. + ('https://localhost:3081/opendev/system-config', None, + 'gitea-project-system-config2.png'), ('https://localhost:3081/opendev/disk-image-builder', None, 'gitea-project-dib.png'), ('https://localhost:3081/opendev/', None, @@ -150,3 +161,11 @@ def test_gitea_screenshots(host): take_screenshots(host, shots) +def test_memcached_has_data(host): + # We only listen on localhost on the gitea server otherwise we could + # open a socket directly from this test case and request the stats data. + cmd = host.run("echo 'stats' | nc -N 127.0.0.1 11211") + assert cmd.succeeded + # Having more than 0 bytes in memcached implies gitea is using + # the server as a cache. + assert 'STAT bytes 0' not in cmd.stdout diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 56af23de58..cfba552c29 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -651,7 +651,7 @@ parent: system-config-run-containers description: | Run the playbook for the gitea servers. - timeout: 4800 + timeout: 5400 nodeset: nodes: - <<: *bridge_node_x86