storage: Deploy read-only instance and migrate services to use read-write/read-only storages depending on use cases
This matches what's been done for the objstorage deployment template.
Review commit by commit:
- first commit is internal detail (it inlines the deployment helper into a deployment.yaml directly as it was done for the objstorage)
- second commit adds a new read-only storage instance (using a read-only objstorage)
- third commit migrates the read-write storage to use the read-write objstorage instance
- last commit migrates remaining services to use the proper read-write/read-only storages.
make swh-helm-diff
[swh] Comparing changes between branches production and deploy-storage-read-write-and-read-only (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment staging...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment staging...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment production...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment production...
[swh] Generate config in deploy-storage-read-write-and-read-only branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.4xgRc3a7/staging-swh.before 2023-12-04 11:14:22.712979426 +0100
+++ /tmp/swh-chart.swh.4xgRc3a7/staging-swh.after 2023-12-04 11:14:23.576979538 +0100
@@ -184,21 +184,21 @@
name: checker-deposit-template
namespace: swh
data:
config.yml.template: |
extraction_dir: "/tmp/swh.checker.deposit/"
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_modules:
- swh.deposit.loader.tasks
task_queues:
- swh.deposit.loader.tasks.ChecksDepositTsk
deposit:
url: https://deposit-dynamic.internal.staging.swh.network/1/private/
auth:
@@ -259,21 +259,21 @@
name: graphql
namespace: swh
data:
# TODO: rename to not have a dot in the name to allow testing
config.yml: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
search:
cls: remote
url: http://search-rpc-ingress
debug: true
introspection: true
max_raw_content_size: 10000
@@ -347,21 +347,21 @@
metadata:
name: indexer-origin-intrinsic-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
indexer_storage:
cls: remote
url: http://indexer-storage-rpc-ingress
objstorage:
cls: remote
url: http://storage1.internal.staging.swh.network:5003/
journal:
@@ -431,21 +431,21 @@
metadata:
name: lister-arch-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.arch.tasks.ArchListerTask
sentry_settings_for_celery_tasks:
@@ -524,21 +524,21 @@
metadata:
name: lister-aur-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.aur.tasks.AurListerTask
sentry_settings_for_celery_tasks:
@@ -617,21 +617,21 @@
metadata:
name: lister-bioconductor-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.bioconductor.tasks.BioconductorListerTask
- swh.lister.bioconductor.tasks.BioconductorIncrementalListerTask
@@ -711,21 +711,21 @@
metadata:
name: lister-bitbucket-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.bitbucket.tasks.IncrementalBitBucketLister
- swh.lister.bitbucket.tasks.FullBitBucketRelister
@@ -805,21 +805,21 @@
metadata:
name: lister-bower-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.bower.tasks.BowerListerTask
sentry_settings_for_celery_tasks:
@@ -898,21 +898,21 @@
metadata:
name: lister-cgit-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.cgit.tasks.CGitListerTask
sentry_settings_for_celery_tasks:
@@ -991,21 +991,21 @@
metadata:
name: lister-cran-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.cran.tasks.CRANListerTask
sentry_settings_for_celery_tasks:
@@ -1084,21 +1084,21 @@
metadata:
name: lister-debian-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.debian.tasks.DebianListerTask
sentry_settings_for_celery_tasks:
@@ -1177,21 +1177,21 @@
metadata:
name: lister-dlang-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.dlang.tasks.DlangListerTask
sentry_settings_for_celery_tasks:
@@ -1270,21 +1270,21 @@
metadata:
name: lister-gitea-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gitea.tasks.IncrementalGiteaLister
- swh.lister.gitea.tasks.RangeGiteaLister
- swh.lister.gitea.tasks.FullGiteaRelister
@@ -1365,21 +1365,21 @@
metadata:
name: lister-gitiles-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gitiles.tasks.GitilesListerTask
sentry_settings_for_celery_tasks:
@@ -1458,21 +1458,21 @@
metadata:
name: lister-gitlab-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gitlab.tasks.IncrementalGitLabLister
- swh.lister.gitlab.tasks.RangeGitLabLister
- swh.lister.gitlab.tasks.FullGitLabRelister
@@ -1553,21 +1553,21 @@
metadata:
name: lister-gitweb-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gitweb.tasks.GitwebListerTask
sentry_settings_for_celery_tasks:
@@ -1646,21 +1646,21 @@
metadata:
name: lister-gnu-full-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gnu.tasks.GNUListerTask
sentry_settings_for_celery_tasks:
@@ -1739,21 +1739,21 @@
metadata:
name: lister-gogs-full-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.gogs.tasks.FullGogsRelister
sentry_settings_for_celery_tasks:
@@ -1832,21 +1832,21 @@
metadata:
name: lister-golang-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.golang.tasks.FullGolangLister
- swh.lister.golang.tasks.IncrementalGolangLister
@@ -1926,21 +1926,21 @@
metadata:
name: lister-launchpad-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.launchpad.tasks.FullLaunchpadLister
- swh.lister.launchpad.tasks.IncrementalLaunchpadLister
@@ -2020,21 +2020,21 @@
metadata:
name: lister-maven-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.maven.tasks.FullMavenLister
- swh.lister.maven.tasks.IncrementalMavenLister
@@ -2114,21 +2114,21 @@
metadata:
name: lister-nixguix-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
extensions_to_ignore:
- rock
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.nixguix.tasks.NixGuixListerTask
@@ -2209,21 +2209,21 @@
metadata:
name: lister-npm-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.npm.tasks.NpmListerTask
sentry_settings_for_celery_tasks:
@@ -2302,21 +2302,21 @@
metadata:
name: lister-opam-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.opam.tasks.OpamListerTask
sentry_settings_for_celery_tasks:
@@ -2395,21 +2395,21 @@
metadata:
name: lister-packagist-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.packagist.tasks.PackagistListerTask
sentry_settings_for_celery_tasks:
@@ -2488,21 +2488,21 @@
metadata:
name: lister-pagure-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.pagure.tasks.PagureListerTask
sentry_settings_for_celery_tasks:
@@ -2581,21 +2581,21 @@
metadata:
name: lister-phabricator-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.phabricator.tasks.FullPhabricatorLister
sentry_settings_for_celery_tasks:
@@ -2674,21 +2674,21 @@
metadata:
name: lister-pubdev-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.pubdev.tasks.PubDevListerTask
sentry_settings_for_celery_tasks:
@@ -2767,21 +2767,21 @@
metadata:
name: lister-pypi-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.pypi.tasks.PyPIListerTask
sentry_settings_for_celery_tasks:
@@ -2860,21 +2860,21 @@
metadata:
name: lister-rpm-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.rpm.tasks.FullRPMLister
- swh.lister.rpm.tasks.IncrementalRPMLister
@@ -2954,21 +2954,21 @@
metadata:
name: lister-sourceforge-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.sourceforge.tasks.FullSourceForgeLister
- swh.lister.sourceforge.tasks.IncrementalSourceForgeLister
@@ -3048,21 +3048,21 @@
metadata:
name: lister-stagit-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_acks_late: true
task_queues:
- swh.lister.stagit.tasks.StagitListerTask
sentry_settings_for_celery_tasks:
@@ -3356,21 +3356,21 @@
metadata:
name: search-journal-client-indexed-configuration-template
namespace: swh
data:
config.yml.template: |
search:
cls: remote
url: http://search-rpc-ingress
storage:
cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
journal:
brokers:
- journal1.internal.staging.swh.network
- journal2.internal.staging.swh.network
group_id: swh.search.journal_client.indexed-v0.11
object_types:
- origin_intrinsic_metadata
- origin_extrinsic_metadata
prefix: swh.journal.indexed
---
@@ -3380,21 +3380,21 @@
metadata:
name: search-journal-client-objects-configuration-template
namespace: swh
data:
config.yml.template: |
search:
cls: remote
url: http://search-rpc-ingress
storage:
cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
journal:
brokers:
- journal1.internal.staging.swh.network
- journal2.internal.staging.swh.network
group_id: swh.search.journal_client-v0.11
object_types:
- origin
- origin_visit_status
prefix: swh.journal.objects
---
@@ -4106,32 +4106,50 @@
error_reporter:
db: 1
host: redis-postgresql.redis
port: 6379
---
# Source: swh/templates/storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
- name: storage-postgresql-configuration-template
+ name: storage-postgresql-read-only-configuration-template
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: retry
+ - cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh password=${POSTGRESQL_PASSWORD}
+ objstorage:
+ cls: remote
+ url: http://objstorage-read-only-rpc-ingress
+---
+# Source: swh/templates/storage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: storage-postgresql-read-write-configuration-template
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: record_references
- cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
objstorage:
cls: remote
- url: http://storage1.internal.staging.swh.network:5003/
+ url: http://objstorage-read-write-rpc-ingress
journal_writer:
cls: kafka
brokers:
- journal1.internal.staging.swh.network
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
client_id: swh.storage.journal_writer.storage1
anonymize: true
producer_config:
@@ -4569,21 +4587,21 @@
metadata:
namespace: swh
name: web-configuration-template
data:
config.yml.template: |
instance_name: webapp-postgresql.internal.staging.swh.network
allowed_hosts:
- webapp-postgresql.internal.staging.swh.network
storage:
cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
search:
cls: remote
url: http://search-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
vault:
cls: remote
url: http://vault-rpc-ingress
indexer_storage:
@@ -14278,21 +14296,31 @@
name: search-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
- name: storage-postgresql-rpc-ingress
+ name: storage-postgresql-read-only-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-postgresql-read-write-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: vault-rpc-ingress
@@ -14400,26 +14428,41 @@
targetPort: 9125
protocol: UDP
- name: http
port: 9102
targetPort: 9102
---
# Source: swh/templates/storage/service.yaml
apiVersion: v1
kind: Service
metadata:
- name: storage-postgresql
+ name: storage-postgresql-read-only
namespace: swh
spec:
type: ClusterIP
selector:
- app: storage-postgresql
+ app: storage-postgresql-read-only
+ ports:
+ - port: 5002
+ targetPort: 5002
+ name: rpc
+---
+# Source: swh/templates/storage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-postgresql-read-write
+ namespace: swh
+spec:
+ type: ClusterIP
+ selector:
+ app: storage-postgresql-read-write
ports:
- port: 5002
targetPort: 5002
name: rpc
---
# Source: swh/templates/web/service.yaml
apiVersion: v1
kind: Service
metadata:
name: web
@@ -14789,21 +14832,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: checker-deposit
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 355f2425403b3c480246f6ea555abc77507397f3f4089bb2b1a167e4f1301647
+ checksum/config: 271d45484e06293d05a9866db72ce2a36fc7a22625f0ed3fd1f5c194b8eab6c9
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/deposit
operator: In
values:
@@ -14950,21 +14993,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: graphql
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: f7dda209d6bb07512e26a523854bf434d895e9b3c2a07a92dd0db48d0b00eef6
+ checksum/config: 71ea4092224c2ef99a8bd6d06660c1263b83663c59ea6f2b420d779a4692c54d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
@@ -15192,21 +15235,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-origin-intrinsic
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: a2ceebb08d978c9bb866a9adaf63e680e73877df2f045217e184b07d0d514292
+ checksum/config: dbd0f0727c4c043751469cf123c2e9abf97d1f99ec1a7be0a555068555cfaf0a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/indexer
operator: In
values:
@@ -15333,21 +15376,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-arch
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: de6162ce1e5aa7c3080ddc5876c7069270c9c52540044ac3eb8411d0ad73ff0b
+ checksum/config: 27942352583427da071b2a945f30aa392de857a666480e28c15903e539343ef2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -15496,21 +15539,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-aur
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5b52eea38d3cf1b693c9dcb36164188b6d285e55ef29586b783f0a83714a5338
+ checksum/config: 84747e65913592269b1628411175a328724007c6a3f33b23430959127841a02b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -15659,21 +15702,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-bioconductor
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 6feea24706981e9ba91ef02a8148b60517e8d7d01cd6fcf3fc9328fec4590a9a
+ checksum/config: 4986200d943ab599462b515f8a675bbd442b83fb90b52fac8e74b7a122266a99
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -15822,21 +15865,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-bitbucket
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 59ae9a3873847449a9d5967f649bcdcb2932734c379b69d522b902f7afcaf73b
+ checksum/config: cfa23e42bb01f734b9d5e56d3bb00b51fdbd6dadd95b2e5c064371dd0a374d2d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -15985,21 +16028,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-bower
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 4f4237a2707e28fdbd4c413e8c44352c25b5bf0eb9daf5f6dd5a1e3d93748324
+ checksum/config: 82a216ee8ffafb6f56cdbbeedaea421898536a4ae48fc3414ea6fcb0bed43cac
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16148,21 +16191,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-cgit
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 66ea414f620d004249a202a41da482f5d505cee9e05345d806ec924d1af79f89
+ checksum/config: a23609fdad1e8fa587dd3942f1a66cb467331bb35b4ff11cf7cfbae1b5a046c3
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16311,21 +16354,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-cran
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: bbb4475baf89f9ad6e9bf44181ff8e451a0bc874ca1dcb4a4ad3e062c022e221
+ checksum/config: d3d1cc1ee3e1e4108f2f7d253ae392c317345203799e8181d08aa505834f4f6e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16474,21 +16517,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-debian
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 00f07514f111bce0158be1b2713f5cafb525f38e7aa4221577467a9083da55c4
+ checksum/config: 19ba620564d3867dee5137df1c19b7288b3c21c0df481bd3be5dd79f3817b9f5
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16637,21 +16680,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-dlang
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 7bed946a8b0eb8abd1019259a1e12aef4766c5eb750d584e066838d39a9f4c94
+ checksum/config: f73e7d0f721dcc9549e16eb93553f0d09b0b3463be33bc050d77eaad9cf46dcc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16800,21 +16843,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gitea
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: ad7b980828e820eb1b2577283e3d1885e7dd45d0d4e694ea3442ef3ce3cbbaa8
+ checksum/config: 0fc93888fa480567c3adbb56c04d035f8b509243559d3504d1eec5bc50f5faee
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -16963,21 +17006,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gitiles
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 6bd07f02e08cee1a854b61ee32113206ae7fe538fe08d33de6c2ee708eff14c8
+ checksum/config: 76acb7d175d8e20ef8ba017e5b7dfc40e175bc3db9ca3e05dc4141eb8efa0c74
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17126,21 +17169,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gitlab
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e326038ebee51378366169ac7a9ca5b5e076b8e5b97ebebd6d02e820af16b7e5
+ checksum/config: 20629043fb05203c3286a68108284b9e7aa5e64b2b8d19a7083d5455e8464d46
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17289,21 +17332,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gitweb
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 246eb1499b02623af5aa90649d27784c16e441c281f95646a2ddb083c8d5b202
+ checksum/config: 45f925bb43765fdad04bf84d3bb0dcffdc67a457e6f3ee09e10b767c63aab9fa
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17452,21 +17495,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gnu-full
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: a975e9777ee05b4bb4be0a3a2b3c40d4587a60ec3f81160bd82048e239fb67b1
+ checksum/config: 0255bb9fce6e9cf7b13222794d092e0daf8885bc2eeb352d79d21dc54a24d6a7
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17615,21 +17658,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gogs-full
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 4504d704ca4a3d5fd7fb8b46da6ad5b4a1da470544f9a60a46b36c6adaf24759
+ checksum/config: 804e67e0343fa641bff9f72b26aadae6e51bbfe6a28261773dcc9a6c5180a8a8
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17778,21 +17821,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-golang
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: d85f60e5f57d0299ab2594e7ea63d8651412c3dd15b43827310c3058368b38ad
+ checksum/config: 78f9564a67603e3d76d3a18891b9c380276653684656d7ed82d458896a170571
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -17941,21 +17984,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-launchpad
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: b92b8b1bf44b829e72fb3d86ae6c1841cc4cde046aed369169bf35b528c5fa88
+ checksum/config: 8f278bbc3fab2ba31428842ef8783bdd9f635437e941caae47cf67569c599168
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18104,21 +18147,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-maven
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: b43dac921cb888db2612d0767ed96d34dcd7fbbf8e01cf654fcd058935399175
+ checksum/config: fe4085e22317806d6e171f73e7fa9f65078807d283e8d853e9c8bae6bd8a8832
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18267,21 +18310,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-nixguix
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 8cc531e3d88111c1b9c8671b83f6cf0c7d02c9aed7d2e9c743f09c21bea48044
+ checksum/config: c228b5e90957371ffaea299cf0e96d1d38dd479795852c686c1795e9b2c71fa2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18430,21 +18473,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-npm
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: c99ea87d186803e0f2754f46ac5f8b0cad0c9362f3dfe84fab9aec4db3e5b46a
+ checksum/config: 74fe9a0d76fbb9dd1a7bbaa930f1a5b5c98a96f16baaf6c5ddbe031fa1efd87c
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18593,21 +18636,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-opam
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 03b4425a4a31ff68878d5fb7ff474f84716d615f60d75f8cd02de1240449d680
+ checksum/config: 6d510f20145949e5a43a366460737366297d695d2598c8717e4320833ac9ceb4
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18756,21 +18799,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-packagist
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 3c16b7b4eada283d18c33cd3b3350200e5cb851f2caf756d6e6cdaf0386ae468
+ checksum/config: a72695ad6a32c48f0178002539a74a7ee6fe9ed95a33cba4e04710aec76e35dd
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -18919,21 +18962,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-pagure
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e52b2ee9c2f3388744a67d920d1631ec60ee1a002a5dec567f55f058043c6c54
+ checksum/config: b85efd4433dc89fdc84107e561de40313acd05fa0ce39753e60fd1b34f142859
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19082,21 +19125,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-phabricator
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: d353405a699676b07900022aeb8d70f49f1ac3e10826eab15c3d0535ebfbdea6
+ checksum/config: 737380111a73fbca4a7a679b58df51dc2b2026e7d813a5c9c0d217af62532d6a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19245,21 +19288,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-pubdev
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5cfdb740253eadf2b67c7b99e8917d4de00d5d3035c836a75b47f09228d3b11a
+ checksum/config: 9bdb0b0f3f08f17a2343d9bf1f87c7aee9a1e061e52393f68e233bbfa40d8295
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19408,21 +19451,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-pypi
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 1bd2fb5ee8c7c9456a0c733d2bf351bd30b936984661e34445aa8e9aafd153d3
+ checksum/config: c7182124d850f466d11ad88109db21c1892cd5d304fd9817f97c58e85bf07e8c
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19571,21 +19614,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-rpm
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 8448ad97a075dd4937517af5fd178b4db1a7c59b1adcac5d7702f98768d0551b
+ checksum/config: 92d9fd5b52727f27f5044e2dd5e5f0dd9658538aa30bad6c4ae896173d86cac9
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19734,21 +19777,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-sourceforge
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 9cdecc4ec3f1eb02bab82792ba591a62ea0dfd1fc36df8aa2fd8b2bb6c0a86f9
+ checksum/config: 8a602e177b2727848c6c37a0524a687aca1cb1333f904172f80ad4e6415dcc74
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -19897,21 +19940,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-stagit
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 9348eabbb9a743478eb9d23d6a1b543aca858d1e10ec44452a7fbf175c3d41e9
+ checksum/config: 07a598f126e6361c495cf55131adae392bde595d267c44ac2b17b308df49ef66
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -21723,21 +21766,21 @@
app: search-journal-client-indexed
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: search-journal-client-indexed
annotations:
- checksum/config: a92ab2218121926d39962c8de7e466d809a8d78db5f3509a740ecadf71ac4df1
+ checksum/config: ea4118b48a38c5ad7df0ce7232693010cb5e24b68c8ea72cc41e550960997afe
spec:
priorityClassName: swh-normal-workload
initContainers:
- name: prepare-configuration
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
@@ -21817,21 +21860,21 @@
app: search-journal-client-objects
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: search-journal-client-objects
annotations:
- checksum/config: 747b935796f4046da313e2bcae7751132588bea355cd868be31c52735b344309
+ checksum/config: dc51b9e1fb428757df53c209492676fff6827144eb7f947fb027dcfa45c287e9
spec:
priorityClassName: swh-normal-workload
initContainers:
- name: prepare-configuration
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
@@ -23585,38 +23628,177 @@
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
---
# Source: swh/templates/storage/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh
- name: storage-postgresql
+ name: storage-postgresql-read-only
+ labels:
+ app: storage-postgresql-read-only
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: storage-postgresql-read-only
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: storage-postgresql-read-only
+ annotations:
+ checksum/config: 33fd736bda9935aa17b76cf03e0709a7d1a716bfb0fb8c22dacedbea20aa543e
+ checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
+ checksum/config-utils: 71310cd853cd229b81e20e236c59bf2801c68fc23784aa3c4f7de2c4e6428476
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/storage
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+
+
+ containers:
+ - name: storage-postgresql-read-only
+ resources:
+ requests:
+ memory: 1500Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage:20231127.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5002
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: THREADS
+ value: "4"
+ - name: WORKERS
+ value: "8"
+ - name: TIMEOUT
+ value: "60"
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: LOG_LEVEL
+ value: "INFO"
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.storage
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: storage-postgresql-read-only-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/storage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh
+ name: storage-postgresql-read-write
labels:
- app: storage-postgresql
+ app: storage-postgresql-read-write
spec:
revisionHistoryLimit: 2
selector:
matchLabels:
- app: storage-postgresql
+ app: storage-postgresql-read-write
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
- app: storage-postgresql
+ app: storage-postgresql-read-write
annotations:
- checksum/config: 3dc03240f4d235973642713fd9be1d8cbaa0a20e1754cf78843dc56baa61761c
+ checksum/config: 20e527c47e6af53e370155ac60b2542c83b836f61e4f37f1d14abca145d18f72
checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
checksum/config-utils: 71310cd853cd229b81e20e236c59bf2801c68fc23784aa3c4f7de2c4e6428476
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
@@ -23643,21 +23825,21 @@
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
containers:
- - name: storage-postgresql
+ - name: storage-postgresql-read-write
resources:
requests:
memory: 1500Mi
cpu: 500m
image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage:20231127.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5002
name: rpc
readinessProbe:
@@ -23706,21 +23888,21 @@
- name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
value: "true"
volumeMounts:
- name: configuration
mountPath: /etc/swh
volumes:
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
- name: storage-postgresql-configuration-template
+ name: storage-postgresql-read-write-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
- name: config-utils
configMap:
name: config-utils
@@ -24039,21 +24221,21 @@
app: web
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: web
annotations:
- checksum/config: 0b66351212a350c2cf8404206e0f3050b6b5f7d91faf26cd99a7459d8d349c38
+ checksum/config: 1145926b7abcb3bfd38c5ba6018b8b729e3609dc0140ce0846c86e65c8d0c9d2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
values:
- "true"
@@ -24321,28 +24503,49 @@
name: cpu
target:
type: Utilization
averageUtilization: 75
---
# Source: swh/templates/storage/autoscaling.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
namespace: swh
- name: storage-postgresql
+ name: storage-postgresql-read-only
+ labels:
+ app: storage-postgresql-read-only
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: storage-postgresql-read-only
+ minReplicas: 2
+ maxReplicas: 10
+ metrics:
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: 150---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ namespace: swh
+ name: storage-postgresql-read-write
labels:
- app: storage-postgresql
+ app: storage-postgresql-read-write
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
- name: storage-postgresql
+ name: storage-postgresql-read-write
minReplicas: 2
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 150
---
@@ -24592,46 +24795,60 @@
service:
name: search-rpc
port:
number: 5010
---
# Source: swh/templates/storage/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
- name: storage-postgresql-ingress-default
+ name: storage-postgresql-read-only-ingress-default
annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: "on"
spec:
rules:
- - host: storage-postgresql-rpc-ingress
+ - host: storage-postgresql-read-only-rpc-ingress
http:
paths:
- path: /
pathType: Prefix
backend:
service:
- name: storage-postgresql
+ name: storage-postgresql-read-only
port:
number: 5002
-
- - host: storage-postgresql.internal.staging.swh.network
+---
+# Source: swh/templates/storage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh
+ name: storage-postgresql-read-write-ingress-default
+ annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.130.0/24,192.168.50.0/24
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+ rules:
+ - host: storage-postgresql-read-write-rpc-ingress
http:
paths:
- path: /
pathType: Prefix
backend:
service:
- name: storage-postgresql
+ name: storage-postgresql-read-write
port:
number: 5002
---
# Source: swh/templates/web/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
name: web-ingress-authenticated
annotations:
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.4xgRc3a7/staging-swh-cassandra.before 2023-12-04 11:14:22.984979461 +0100
+++ /tmp/swh-chart.swh.4xgRc3a7/staging-swh-cassandra.after 2023-12-04 11:14:23.836979572 +0100
@@ -183,21 +183,21 @@
metadata:
name: cooker-batch-template
namespace: swh-cassandra
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
vault:
cls: remote
url: http://vault-rpc-ingress
max_bundle_size: 1073741824
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_modules:
- swh.vault.cooking_tasks
task_queues:
- swh.vault.cooking_tasks.SWHBatchCookingTask
@@ -265,21 +265,21 @@
metadata:
name: cooker-simple-template
namespace: swh-cassandra
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
vault:
cls: remote
url: http://vault-rpc-ingress
max_bundle_size: 1073741824
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_modules:
- swh.vault.cooking_tasks
task_queues:
- swh.vault.cooking_tasks.SWHCookingTask
@@ -4376,31 +4376,31 @@
data:
config.yml.template: |
vault:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-vault dbname=swh-vault password=${POSTGRESQL_PASSWORD}
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
objstorage:
cls: filtered
filters_conf:
- type: readonly
storage_conf:
cls: remote
- url: http://storage1.internal.staging.swh.network:5003
+ url: http://storage-read-only-rpc-ingress
cache:
cls: azure
compression: none
connection_string: DefaultEndpointsProtocol=https;AccountName=swhvaultstoragestaging;AccountKey=${ACCOUNT_KEY};EndpointSuffix=core.windows.net
container_name: contents-uncompressed
smtp:
host: smtp.inria.fr
port: 25
---
# Source: swh/templates/web/configmap-pgservice.yaml
@@ -14174,21 +14174,31 @@
name: search-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
- name: storage-postgresql-rpc-ingress
+ name: storage-postgresql-read-only-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-postgresql-read-write-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: vault-rpc-ingress
@@ -14685,21 +14695,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: cooker-batch
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5e7d546d490162d4b52a322507212b98c94bfba57f23eae9724de0b33553c6d3
+ checksum/config: e137ddf0bf84044d0215890ad3c2c39b7390814ddbe3d3c39d786a164757a12b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/cooker
operator: In
values:
@@ -14828,21 +14838,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: cooker-simple
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5e7d546d490162d4b52a322507212b98c94bfba57f23eae9724de0b33553c6d3
+ checksum/config: e137ddf0bf84044d0215890ad3c2c39b7390814ddbe3d3c39d786a164757a12b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/cooker
operator: In
values:
@@ -22414,21 +22424,21 @@
app: vault-rpc
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: vault-rpc
annotations:
- checksum/config: 110679eaefa29bcadbeb5dfe3e91c3db4d56752696feb66d9fed921291595f93
+ checksum/config: 96ad47ae98446630bbeafb66ee0d4cc15d5d3f0ecfc52483c3cd88b1b7107921
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.4xgRc3a7/staging-swh-cassandra-next-version.before 2023-12-04 11:14:23.284979500 +0100
+++ /tmp/swh-chart.swh.4xgRc3a7/staging-swh-cassandra-next-version.after 2023-12-04 11:14:24.096979606 +0100
@@ -183,21 +183,21 @@
metadata:
name: cooker-batch-template
namespace: swh-cassandra-next-version
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
vault:
cls: remote
url: http://vault-rpc-ingress-next-version
max_bundle_size: 1073741824
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_modules:
- swh.vault.cooking_tasks
task_queues:
- swh.vault.cooking_tasks.SWHBatchCookingTask
@@ -265,21 +265,21 @@
metadata:
name: cooker-simple-template
namespace: swh-cassandra-next-version
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
vault:
cls: remote
url: http://vault-rpc-ingress-next-version
max_bundle_size: 1073741824
celery:
task_broker: amqp://swhconsumer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
task_modules:
- swh.vault.cooking_tasks
task_queues:
- swh.vault.cooking_tasks.SWHCookingTask
@@ -3904,31 +3904,31 @@
data:
config.yml.template: |
vault:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-vault dbname=swh-vault password=${POSTGRESQL_PASSWORD}
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
- url: http://storage1.internal.staging.swh.network:5002
+ url: http://storage-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
objstorage:
cls: filtered
filters_conf:
- type: readonly
storage_conf:
cls: remote
- url: http://storage1.internal.staging.swh.network:5003
+ url: http://storage-read-only-rpc-ingress
cache:
cls: azure
compression: none
connection_string: DefaultEndpointsProtocol=https;AccountName=swhvaultstoragestaging;AccountKey=${ACCOUNT_KEY};EndpointSuffix=core.windows.net
container_name: contents-uncompressed
smtp:
host: smtp.inria.fr
port: 25
---
# Source: swh/templates/web/configmap.yaml
@@ -13647,21 +13647,31 @@
name: search-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
- name: storage-postgresql-rpc-ingress
+ name: storage-postgresql-read-only-rpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-postgresql-read-write-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: vault-rpc-ingress
@@ -14128,21 +14138,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: cooker-batch
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5f309b878310155ec483585a8a159eb594a6517cfc3de9aaa48c1d9a028142c1
+ checksum/config: 6554b2213c79625e8111f9c4d7a8cea2c7e6e0fe674a5c6bbc21b6b657f61e25
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/cooker
operator: In
values:
@@ -14271,21 +14281,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: cooker-simple
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 5f309b878310155ec483585a8a159eb594a6517cfc3de9aaa48c1d9a028142c1
+ checksum/config: 6554b2213c79625e8111f9c4d7a8cea2c7e6e0fe674a5c6bbc21b6b657f61e25
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/cooker
operator: In
values:
@@ -20591,21 +20601,21 @@
app: vault-rpc
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: vault-rpc
annotations:
- checksum/config: c4a7195c02d61774900f3ad1c93c5f728e45f5212034329bb34db811c3d78bfb
+ checksum/config: 5fd1276ea58285267703a2fb6d22a635db8b648df75bdd7d72d49e687d7d132d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differencesEdited by Antoine R. Dumont