[Add Forge Now] process https://gitlab.liu.se

https://archive.softwareheritage.org/admin/add-forge/request/808/
Type: gitlab

changed milestone to %Extend archive coverage [Roadmap - Collect]

added AddForgeNow label

Request completly messed up in staging environment.
For the pipeline details, see:
https://gitlab.softwareheritage.org/swh/infra/add-forge-now-requests/-/pipelines/11277.

GitLab lister got an error from the forge:

lister-gitlab-8b8f898b7-kc754 listers {"asctime": "2024-10-09 16:37:23,001", "threadName": "MainThread", "pathname": "/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py", "lineno": 161, "funcName": "get_page_result", "task_name": null, "task_id": null, "name": "swh.lister.gitlab.lister", "levelname": "WARNING", "message": "Unexpected HTTP status code 500 on https://gitlab.liu.se/api/v4/projects?pagination=keyset&order_by=id&sort=asc&simple=true&per_page=100: b'{\"message\":\"500 Internal Server Error\"}'"}

I will try to update the lister task (status,next_run) to relaunch it.

GitLab lister got the same error:

ᐅ stern --context archive-staging-rke2 -n swh-cassandra -l app=lister-gitlab --only-log-lines --no-follow --tail 3
lister-gitlab-8b8f898b7-rwh25 listers {"asctime": "2024-10-10 07:46:59,641", "threadName": "MainThread", "pathname": "/opt/swh/.local/lib/python3.10/site-packages/celery/worker/strategy.py", "lineno": 161, "funcName": "task_message_handler", "task_name": null, "task_id": null, "name": "celery.worker.strategy", "levelname": "INFO", "message": "Task swh.lister.gitlab.tasks.FullGitLabRelister[230eb097-31e9-4f13-b0f0-054c87ae8405] received", "data": {"id": "230eb097-31e9-4f13-b0f0-054c87ae8405", "name": "swh.lister.gitlab.tasks.FullGitLabRelister", "args": "()", "kwargs": "{'instance': 'gitlab.liu.se', 'max_pages': 2, 'enable_origins': False, 'max_origins_per_page': 5}", "eta": null}}
lister-gitlab-8b8f898b7-rwh25 listers {"asctime": "2024-10-10 07:47:07,995", "threadName": "MainThread", "pathname": "/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py", "lineno": 161, "funcName": "get_page_result", "task_name": null, "task_id": null, "name": "swh.lister.gitlab.lister", "levelname": "WARNING", "message": "Unexpected HTTP status code 500 on https://gitlab.liu.se/api/v4/projects?pagination=keyset&order_by=id&sort=asc&simple=true&per_page=100: b'{\"message\":\"500 Internal Server Error\"}'"}
lister-gitlab-8b8f898b7-rwh25 listers {"asctime": "2024-10-10 07:47:08,022", "threadName": "MainThread", "pathname": "/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py", "lineno": 267, "funcName": "_log_error", "task_name": null, "task_id": null, "name": "celery.app.trace", "levelname": "ERROR", "message": "Task swh.lister.gitlab.tasks.FullGitLabRelister[230eb097-31e9-4f13-b0f0-054c87ae8405] raised unexpected: AssertionError()", "exc_info": "Traceback (most recent call last):\n  File \"/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py\", line 453, in trace_task\n    R = retval = fun(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/utils.py\", line 1761, in runner\n    return sentry_patched_function(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/integrations/celery/__init__.py\", line 414, in _inner\n    reraise(*exc_info)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/utils.py\", line 1690, in reraise\n    raise value\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/integrations/celery/__init__.py\", line 409, in _inner\n    return f(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/scheduler/task.py\", line 61, in __call__\n    result = super().__call__(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py\", line 736, in __protected_call__\n    return self.run(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/tasks.py\", line 21, in list_gitlab_full\n    return lister.run().dict()\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/pattern.py\", line 219, in run\n    for page in self.get_pages():\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py\", line 215, in get_pages\n    page_result = self.get_page_result(next_page)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 336, in wrapped_f\n    return copy(f, *args, **kw)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 475, in __call__\n    do = self.iter(retry_state=retry_state)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 376, in iter\n    result = action(retry_state)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 398, in <lambda>\n    self._add_action_func(lambda rs: rs.outcome.result())\n  File \"/usr/local/lib/python3.10/concurrent/futures/_base.py\", line 451, in result\n    return self.__get_result()\n  File \"/usr/local/lib/python3.10/concurrent/futures/_base.py\", line 403, in __get_result\n    raise self._exception\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 478, in __call__\n    result = fn(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py\", line 174, in get_page_result\n    assert id_after is not None\nAssertionError", "data": {"hostname": "lister@lister-gitlab-8b8f898b7-rwh25", "id": "230eb097-31e9-4f13-b0f0-054c87ae8405", "name": "swh.lister.gitlab.tasks.FullGitLabRelister", "exc": "AssertionError()", "traceback": "Traceback (most recent call last):\n  File \"/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py\", line 453, in trace_task\n    R = retval = fun(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/utils.py\", line 1761, in runner\n    return sentry_patched_function(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/integrations/celery/__init__.py\", line 414, in _inner\n    reraise(*exc_info)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/utils.py\", line 1690, in reraise\n    raise value\n  File \"/opt/swh/.local/lib/python3.10/site-packages/sentry_sdk/integrations/celery/__init__.py\", line 409, in _inner\n    return f(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/scheduler/task.py\", line 61, in __call__\n    result = super().__call__(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/celery/app/trace.py\", line 736, in __protected_call__\n    return self.run(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/tasks.py\", line 21, in list_gitlab_full\n    return lister.run().dict()\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/pattern.py\", line 219, in run\n    for page in self.get_pages():\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py\", line 215, in get_pages\n    page_result = self.get_page_result(next_page)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 336, in wrapped_f\n    return copy(f, *args, **kw)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 475, in __call__\n    do = self.iter(retry_state=retry_state)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 376, in iter\n    result = action(retry_state)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 398, in <lambda>\n    self._add_action_func(lambda rs: rs.outcome.result())\n  File \"/usr/local/lib/python3.10/concurrent/futures/_base.py\", line 451, in result\n    return self.__get_result()\n  File \"/usr/local/lib/python3.10/concurrent/futures/_base.py\", line 403, in __get_result\n    raise self._exception\n  File \"/opt/swh/.local/lib/python3.10/site-packages/tenacity/__init__.py\", line 478, in __call__\n    result = fn(*args, **kwargs)\n  File \"/opt/swh/.local/lib/python3.10/site-packages/swh/lister/gitlab/lister.py\", line 174, in get_page_result\n    assert id_after is not None\nAssertionError\n", "args": "()", "kwargs": "{'instance': 'gitlab.liu.se', 'max_pages': 2, 'enable_origins': False, 'max_origins_per_page': 5}", "description": "raised unexpected", "internal": false}}

Removing the sort=asc& from the URL makes it work, but unfortunately that is currently hard-coded in the code.