Skip to content

loaders: Move retry proxy to the bottom of the stack

Antoine R. Dumont requested to merge generated-differential-D5330-source into generated-differential-D5330-target

There is no advantage to retry at the outer level. As the operations performed should mostly succeed, retrying those ops would be more work for noop result. Especially, the filter part which would make the backend compute some more for noop.

So moving the retry part at the bottom would avoid those inconveniences.

Test Plan

octo:

bin/octocatalog-diff --octocatalog-diff-args --no-truncate-details --to staging worker01
Found host worker01.softwareheritage.org
WARN     -> Environment "wip-pg-hba-rules-in-yaml" contained non-word characters, correcting name to wip_pg_hba_rules_in_yaml
Cloning into '/tmp/swh-ocd.pGp6iO4k/environments/production/data/private'...
done.
Cloning into '/tmp/swh-ocd.pGp6iO4k/environments/staging/data/private'...
done.
*** Running octocatalog-diff on host worker01.softwareheritage.org
I, [2021-03-25T11:53:23.419622 #18542]  INFO -- : Catalogs compiled for worker01.softwareheritage.org
I, [2021-03-25T11:53:24.476583 #18542]  INFO -- : Diffs computed for worker01.softwareheritage.org
diff origin/production/worker01.softwareheritage.org current/worker01.softwareheritage.org
*******************************************
  File[/etc/softwareheritage/loader_archive.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_cran.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_debian.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_deposit.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_git.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_mercurial.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_nixguix.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_npm.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_pypi.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
  File[/etc/softwareheritage/loader_svn.yml] =>
   parameters =>
     content =>
      @@ -3,5 +3,4 @@
         cls: pipeline
         steps:
      -  - cls: retry
         - cls: buffer
           min_batch_size:
      @@ -12,4 +11,5 @@
             release: 1000
         - cls: filter
      +  - cls: retry
         - cls: remote
           args:
*******************************************
*** End octocatalog-diff on worker01.softwareheritage.org

Migrated from D5330 (view on Phabricator)

Merge request reports