storage: ensure autoscaling is properly configured for all storages
- Use aws as main objstorage backend
- swh/production: Remove retry from read-only storage rpc backend
- Add the support for empty pipeline
helm diff
[swh] Comparing changes between branches staging and storage-aws-first (per environment)...
Switched to branch 'staging'
Your branch is up to date with 'origin/staging'.
[swh] Generate config in staging branch for environment staging, namespace swh...
[swh] Generate config in staging branch for environment staging, namespace swh-cassandra...
[swh] Generate config in staging branch for environment staging, namespace swh-cassandra-next-version...
Switched to branch 'storage-aws-first'
Your branch is up to date with 'origin/storage-aws-first'.
[swh] Generate config in storage-aws-first branch for environment staging...
[swh] Generate config in storage-aws-first branch for environment staging...
[swh] Generate config in storage-aws-first branch for environment staging...
Switched to branch 'staging'
Your branch is up to date with 'origin/staging'.
[swh] Generate config in staging branch for environment production, namespace swh...
[swh] Generate config in staging branch for environment production, namespace swh-cassandra...
[swh] Generate config in staging branch for environment production, namespace swh-cassandra-next-version...
Switched to branch 'storage-aws-first'
Your branch is up to date with 'origin/storage-aws-first'.
[swh] Generate config in storage-aws-first branch for environment production...
[swh] Generate config in storage-aws-first branch for environment production...
[swh] Generate config in storage-aws-first branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.2Dlh0Gym/production-swh.before 2024-01-26 14:05:57.840723162 +0100
+++ /tmp/swh-chart.swh.2Dlh0Gym/production-swh.after 2024-01-26 14:05:58.164727516 +0100
@@ -5992,34 +5992,32 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: objstorage-replayer-winery-template
data:
config.yml.template: |
objstorage:
cls: multiplexer
objstorages:
- - cls: filtered
- filters_conf:
- - type: readonly
- storage_conf:
- cls: http
- compression: gzip
- timeout: 120
- url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: http
+ compression: gzip
+ timeout: 120
+ url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: remote
+ url: http://objstorage-ro-saam-zfs-rpc-ingress
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
cls: remote
- url: http://objstorage-read-only-rpc-ingress
+ url: http://banco.internal.softwareheritage.org:5003/
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
accounts:
"0":
account_name: ${ACCOUNT_NAME_0}
api_secret_key: ${API_SECRET_KEY_0}
container_name: contents
"1":
@@ -6573,26 +6571,37 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: storage-postgresql-azure-readonly-configuration-template
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- - cls: retry
- cls: postgresql
db: host=postgresql-storage-rw.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
objstorage:
cls: multiplexer
objstorages:
+ - cls: http
+ compression: gzip
+ timeout: 120
+ url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: remote
+ url: http://objstorage-ro-saam-zfs-rpc-ingress
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ cls: remote
+ url: http://banco.internal.softwareheritage.org:5003/
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
accounts:
"0":
account_name: ${ACCOUNT_NAME_0}
api_secret_key: ${API_SECRET_KEY_0}
container_name: contents
"1":
@@ -6649,26 +6658,20 @@
container_name: contents
e:
account_name: ${ACCOUNT_NAME_14}
api_secret_key: ${API_SECRET_KEY_14}
container_name: contents
f:
account_name: ${ACCOUNT_NAME_15}
api_secret_key: ${API_SECRET_KEY_15}
container_name: contents
cls: azure-prefixed
- - cls: filtered
- filters_conf:
- - type: readonly
- storage_conf:
- cls: remote
- url: http://objstorage.internal.softwareheritage.org:5003/
---
# Source: swh/templates/storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: storage-postgresql-saam-zfs-configuration-template
data:
config.yml.template: |
storage:
@@ -7237,34 +7240,32 @@
steps:
- cls: retry
- cls: remote
url: http://storage-azure-read-only-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.softwareheritage.org
objstorage:
cls: multiplexer
objstorages:
- - cls: filtered
- filters_conf:
- - type: readonly
- storage_conf:
- cls: http
- compression: gzip
- timeout: 120
- url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: http
+ compression: gzip
+ timeout: 120
+ url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: remote
+ url: http://objstorage-ro-saam-zfs-rpc-ingress
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
cls: remote
- url: http://objstorage-read-only-rpc-ingress
+ url: http://banco.internal.softwareheritage.org:5003/
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
accounts:
"0":
account_name: ${ACCOUNT_NAME_0}
api_secret_key: ${API_SECRET_KEY_0}
container_name: contents
"1":
@@ -18583,21 +18584,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: objstorage-replayer-winery
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: d52b60573ff13740b08f1cf7e0cede1dcd38a62905cf178435214f3008f33aee
+ checksum/config: 58d3b28224e34d10e49de8cc780e218d20031716bb5f7fbb42d858877e8f8894
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
@@ -21793,21 +21794,21 @@
app: storage-postgresql-azure-readonly
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-postgresql-azure-readonly
annotations:
- checksum/config: 98a3428ae9dae1e769de8acd2fcfd46dc28d850b57ba08bf37300c2a8ad1623e
+ checksum/config: ffb50a230a560c35a01ecf8877bca2d7959425d9ab8062d08bc22418657d4e94
checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
@@ -22878,21 +22879,21 @@
app: vault-rpc
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: vault-rpc
annotations:
- checksum/config: 95d36d9d8dc4e174d79254a6c7c950abcfe36edbcb3df836cb367f05ee752eeb
+ checksum/config: d098d1914675d83390a4810f3cbb99ed15e02eadebc316f64ef184a0d0620db3
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.2Dlh0Gym/production-swh-cassandra.before 2024-01-26 14:05:57.928724345 +0100
+++ /tmp/swh-chart.swh.2Dlh0Gym/production-swh-cassandra.after 2024-01-26 14:05:58.252728699 +0100
@@ -1022,34 +1022,32 @@
keyspace: swh
consistency_level: LOCAL_QUORUM
auth_provider:
cls: cassandra.auth.PlainTextAuthProvider
password: ${CASSANDRA_PASSWORD}
username: swh-ro
objstorage:
cls: multiplexer
objstorages:
- - cls: filtered
- filters_conf:
- - type: readonly
- storage_conf:
- cls: http
- compression: gzip
- timeout: 120
- url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: http
+ compression: gzip
+ timeout: 120
+ url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: remote
+ url: http://objstorage-ro-saam-zfs-rpc-ingress
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
cls: remote
- url: http://objstorage-read-only-rpc-ingress
+ url: http://banco.internal.softwareheritage.org:5003/
- cls: filtered
filters_conf:
- type: readonly
storage_conf:
accounts:
"0":
account_name: ${ACCOUNT_NAME_0}
api_secret_key: ${API_SECRET_KEY_0}
container_name: contents
"1":
@@ -4306,21 +4304,21 @@
app: storage-cassandra-readonly
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-cassandra-readonly
annotations:
- checksum/config: 796384ef18da68b142f6ab7936ad2bd770368a255c654162748ed8f7e21692a3
+ checksum/config: 90e169b24b1f8947024341fa6019b641cd5343ad074d2447f25f95c24b561c4d
checksum/database-utils: 367f28d8854b3b188d7a45015752b10be8e691f0c3b2a21db02501e655b932f0
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In