Investigate possible loaders's temp directory cleaning issue
For example, for a given loader-save-code-now-6f58dbd859-9tjc2
save-code-now pod.
Its last log is:
2024-10-24T13:02:53.144765442Z loaders {"asctime": "2024-10-24 13:02:53,144", "threadName": "MainThread", "pathname": "/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py", "lineno": 128, "funcName": "info", "task_name": null, "task_id": null, "name": "celery.app.trace", "levelname": "INFO", "message": "Task swh.loader.git.tasks.UpdateGitRepository[9905e2bc-3a7c-4df3-bbc3-0b4452991913] succeeded in 0.7586854621767998s: {'status': 'uneventful'}", "data": {"id":
"9905e2bc-3a7c-4df3-bbc3-0b4452991913", "name": "swh.loader.git.tasks.UpdateGitRepository", "return_value": "{'status': 'uneventful'}", "runtime": 0.7586854621767998, "args": "()", "kwargs": "{'url': 'https://github.com/Meettya/react'}"}}
but the content of its ephemeral in-memory volume is:
root@rancher-node-metal03:/tmp/k8s-ephemeral-storage/pvc-4ba18491-a834-4d54-99c1-b96c47df32d8_swh-cassandra_loader-save-code-now-6f58dbd859-9tjc2-localstorage# ls -al
total 0
drwxrwxrwx 8 root root 160 24 oct. 11:27 .
drwxr-xr-x 33 root root 660 24 oct. 13:03 ..
drwx------ 2 zack zack 40 24 oct. 11:27 pymp-01925_9j
drwx------ 2 zack zack 40 23 oct. 22:21 pymp-2hfn7hfc
drwx------ 2 zack zack 40 21 oct. 14:50 pymp-ntm393eq
drwx------ 3 zack zack 60 23 oct. 03:40 swh.loader.bzr.from_diskdn4b8_pu-47
drwx------ 4 zack zack 80 24 oct. 08:54 swh.loader.svn.2fck156a-18
drwx------ 5 zack zack 100 24 oct. 08:39 swh.loader.svn.y2joy4xl-18
root@rancher-node-metal03:/tmp/k8s-ephemeral-storage/pvc-4ba18491-a834-4d54-99c1-b96c47df32d8_swh-cassandra_loader-save-code-now-6f58dbd859-9tjc2-localstorage# du -sch *
0 pymp-01925_9j
0 pymp-2hfn7hfc
0 pymp-ntm393eq
140K swh.loader.bzr.from_diskdn4b8_pu-47
20G swh.loader.svn.2fck156a-18
2,3G swh.loader.svn.y2joy4xl-18
22G total
root@rancher-node-metal03:/tmp/k8s-ephemeral-storage/pvc-4ba18491-a834-4d54-99c1-b96c47df32d8_swh-cassandra_loader-save-code-now-6f58dbd859-9tjc2-localstorage# du --inodes -sch *
1 pymp-01925_9j
1 pymp-2hfn7hfc
1 pymp-ntm393eq
25 swh.loader.bzr.from_diskdn4b8_pu-47
970K swh.loader.svn.2fck156a-18
95K swh.loader.svn.y2joy4xl-18
1,1M total