Skip to content

opam: Install and maintain up-to-date shared opam root directories

Antoine R. Dumont requested to merge generated-differential-D6305-source into generated-differential-D6305-target

(on workers for opam lister and loader services)

This shall reduce the disk use to one opam root per worker (instead of worst case scenario of x distincts opam root * y distincts instances, with x the number of workers, y the unknown number of distinct opam instances).

Using /tmp/opam as default sounds sensible enough (providing we make sure that it's initialized at boot time).

(then other diffs are required on the lister/loader for this to actually work [1] [2] [3]. They are adaptations for the lister and loader to actually use correctcly such opam root).

Related to T3590

Test Plan

octo-diff on worker staging node (opam loader deployed only there for now):

+ File[/etc/systemd/system/opam-manage-shared-state-ocamlbench-repo.service] =>
   parameters =>
      "ensure": "file"
      "group": "root"
      "mode": "0444"
      "notify": "Class[Systemd::Systemctl::Daemon_reload]"
      "owner": "root"
      "show_diff": true
      "content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://github.com/OCamlPro/ocamlbench-repo.git && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam repository add --root /tmp/opam/ ocamlbench-repo https://github.com/OCamlPro/ocamlbench-repo.git

[Install]
WantedBy=multi-user.target
<<<
*******************************************
+ File[/etc/systemd/system/opam-manage-shared-state-ocamlbench-repo.timer] =>
   parameters =>
      "ensure": "file"
      "group": "root"
      "mode": "0444"
      "notify": ["Class[Systemd::Systemctl::Daemon_reload]", "Service[opam-manage-shared-state-ocamlbench-repo.timer]"]
      "owner": "root"
      "show_diff": true
      "content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-ocamlbench-repo.service

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo) timer
<<<
*******************************************
+ File[/etc/systemd/system/opam-manage-shared-state-opam.service] =>
   parameters =>
      "ensure": "file"
      "group": "root"
      "mode": "0444"
      "notify": "Class[Systemd::Systemctl::Daemon_reload]"
      "owner": "root"
      "show_diff": true
      "content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (opam)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://opam.ocaml.org && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam init --reinit --bare --no-setup --root /tmp/opam/ opam https://opam.ocaml.org

[Install]
WantedBy=multi-user.target
<<<
*******************************************
+ File[/etc/systemd/system/opam-manage-shared-state-opam.timer] =>
   parameters =>
      "ensure": "file"
      "group": "root"
      "mode": "0444"
      "notify": ["Class[Systemd::Systemctl::Daemon_reload]", "Service[opam-manage-shared-state-opam.timer]"]
      "owner": "root"
      "show_diff": true
      "content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-opam.service

[Unit]
Description=Software Heritage Manage OPAM shared state (opam) timer
<<<
*******************************************
+ File[/tmp/opam] =>
   parameters =>
      "ensure": "directory"
      "mode": "0644"
      "recurse": true
*******************************************
+ Service[opam-manage-shared-state-ocamlbench-repo.timer] =>
   parameters =>
      "enable": true
      "provider": "systemd"
*******************************************
+ Service[opam-manage-shared-state-opam.timer] =>
   parameters =>
      "enable": true
      "provider": "systemd"
*******************************************
+ Systemd::Timer[opam-manage-shared-state-ocamlbench-repo.timer] =>
   parameters =>
      "enable": true
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "service_content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://github.com/OCamlPro/ocamlbench-repo.git && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam repository add --root /tmp/opam/ ocamlbench-repo https://github.com/OCamlPro/ocamlbench-repo.git

[Install]
WantedBy=multi-user.target
<<<
      "timer_content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-ocamlbench-repo.service

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo) timer
<<<
*******************************************
+ Systemd::Timer[opam-manage-shared-state-opam.timer] =>
   parameters =>
      "enable": true
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "service_content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (opam)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://opam.ocaml.org && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam init --reinit --bare --no-setup --root /tmp/opam/ opam https://opam.ocaml.org

[Install]
WantedBy=multi-user.target
<<<
      "timer_content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-opam.service

[Unit]
Description=Software Heritage Manage OPAM shared state (opam) timer
<<<
*******************************************
+ Systemd::Unit_file[opam-manage-shared-state-ocamlbench-repo.service] =>
   parameters =>
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://github.com/OCamlPro/ocamlbench-repo.git && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam repository add --root /tmp/opam/ ocamlbench-repo https://github.com/OCamlPro/ocamlbench-repo.git

[Install]
WantedBy=multi-user.target
<<<
*******************************************
+ Systemd::Unit_file[opam-manage-shared-state-ocamlbench-repo.timer] =>
   parameters =>
      "enable": true
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-ocamlbench-repo.service

[Unit]
Description=Software Heritage Manage OPAM shared state (ocamlbench-repo) timer
<<<
*******************************************
+ Systemd::Unit_file[opam-manage-shared-state-opam.service] =>
   parameters =>
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "content": >>>
# Opam manage shared state unit file
# Managed by puppet class profile::swh::deploy::worker::loader_opam
# Changes will be overwritten

[Unit]
Description=Software Heritage Manage OPAM shared state (opam)
Requires=network.target
After=network.target

[Service]
User=
Group=
Type=simple
ExecStart=( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://opam.ocaml.org && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam init --reinit --bare --no-setup --root /tmp/opam/ opam https://opam.ocaml.org

[Install]
WantedBy=multi-user.target
<<<
*******************************************
+ Systemd::Unit_file[opam-manage-shared-state-opam.timer] =>
   parameters =>
      "enable": true
      "ensure": "present"
      "group": "root"
      "mode": "0444"
      "owner": "root"
      "path": "/etc/systemd/system"
      "show_diff": true
      "content": >>>
[Install]
WantedBy=timers.target

[Timer]
OnCalendar=daily
AccuracySec=4h
Persistent=true
Unit=opam-manage-shared-state-opam.service

[Unit]
Description=Software Heritage Manage OPAM shared state (opam) timer
<<<
*******************************************
*** End octocatalog-diff on worker0.internal.staging.swh.network

Here are the commands that are actually executed by the service for each instance. They are running as expected (from a scratch opam root directory):

    1. no opam instance then install it
    1. if opam instance already installed, just update it
    1. if ocamlbench-pro not present, just add it to the same opam root directory
    1. if ocamlbench-pro present, just update it):
$ rm -rf /tmp/opam
$ ( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://opam.ocaml.org && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam init --reinit --bare --no-setup --root /tmp/opam/ opam https://opam.ocaml.org
[ERROR] Opam has not been initialised, please run `opam init'
[NOTE] Will configure from built-in defaults.
Checking for available remotes: rsync and local, git, mercurial, darcs. Perfect!

<><> Fetching repository information ><><><><><><><><><><><><><><><><><><><><><>
[opam] Initialised
$ ( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://opam.ocaml.org && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam init --reinit --bare --no-setup --root /tmp/opam/ opam https://opam.ocaml.org
opam         https://opam.ocaml.org <default>(1)

<><> Updating package repositories ><><><><><><><><><><><><><><><><><><><><><><>
[opam] no changes from https://opam.ocaml.org
$ ( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://github.com/OCamlPro/ocamlbench-repo.git && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam repository add --root /tmp/opam/ ocamlbench-repo https://github.com/OCamlPro/ocamlbench-repo.git
[WARNING] No switch is currently set, perhaps you meant '--set-default'?
[ocamlbench-repo] Initialised
[ERROR] No switch is currently set. Please use 'opam switch' to set or install a switch
$ ( /usr/bin/opam repo --all --root /tmp/opam/ | grep https://github.com/OCamlPro/ocamlbench-repo.git && /usr/bin/opam update --root /tmp/opam/ ) || /usr/bin/opam repository add --root /tmp/opam/ ocamlbench-repo https://github.com/OCamlPro/ocamlbench-repo.git
ocamlbench-repo git+https://github.com/OCamlPro/ocamlbench-repo.git

<><> Updating package repositories ><><><><><><><><><><><><><><><><><><><><><><>
[opam] no changes from https://opam.ocaml.org
$ /usr/bin/opam repo --all --root /tmp/opam/
# Repository    # Url                                               # Switches(rank)
ocamlbench-repo git+https://github.com/OCamlPro/ocamlbench-repo.git
opam            https://opam.ocaml.org                              <default>(1)

Migrated from D6305 (view on Phabricator)

Merge request reports