Skip to content
Snippets Groups Projects
Commit e3a31ba7 authored by Antoine Lambert's avatar Antoine Lambert
Browse files

api/private/deposit_list: Add new endpoint to be used with datatables

Add new private API endpoint /1/private/deposits/datatables/ to list and
filter deposits whose responses are intended to be consumed by datatables
javascript framework used in deposits admin Web UI.

Originally that view was implemented in swh-web but for performance reasons
it has been decided to move it in swh-deposit, swh-web will then simply
forward the HTTP request to swh-deposit.

Related to T3128
parent 88ea58cb
No related branches found
No related tags found
No related merge requests found
......@@ -3,15 +3,42 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from typing import Any, Dict
from xml.etree import ElementTree
from django.conf import settings
from django.core.paginator import Paginator
from django.db.models import CharField, Q, TextField
from django.http import JsonResponse
from rest_framework.decorators import (
api_view,
authentication_classes,
permission_classes,
)
from rest_framework.generics import ListAPIView
from rest_framework.permissions import AllowAny
from rest_framework.request import Request
import sentry_sdk
from swh.deposit.api.utils import DefaultPagination, DepositSerializer
from swh.deposit.utils import parse_swh_deposit_origin, parse_swh_metadata_provenance
from swh.model.swhids import QualifiedSWHID
from . import APIPrivateView
from ...models import Deposit
def _enrich_deposit_with_metadata(deposit: Deposit) -> Deposit:
deposit_requests = deposit.depositrequest_set.filter(type="metadata")
deposit_requests = deposit_requests.order_by("-id")
# enrich deposit with raw metadata when we have some
if deposit_requests and len(deposit_requests) > 0:
raw_meta = deposit_requests[0].raw_metadata
if raw_meta:
deposit.set_raw_metadata(raw_meta)
return deposit
class APIList(ListAPIView, APIPrivateView):
"""Deposit request class to list the deposit's status per page.
......@@ -33,15 +60,7 @@ class APIList(ListAPIView, APIPrivateView):
deposits = []
for deposit in page_result:
deposit_requests = deposit.depositrequest_set.filter(
type="metadata"
).order_by("-id")
# enrich deposit with raw metadata when we have some
if deposit_requests and len(deposit_requests) > 0:
raw_meta = deposit_requests[0].raw_metadata
if raw_meta:
deposit.set_raw_metadata(raw_meta)
_enrich_deposit_with_metadata(deposit)
deposits.append(deposit)
return deposits
......@@ -66,3 +85,114 @@ class APIList(ListAPIView, APIPrivateView):
deposits_qs = deposits_qs.exclude(external_id__startswith=exclude_like)
return deposits_qs.order_by("id")
def _deposit_search_query(search_value: str) -> Q:
fields = [f for f in Deposit._meta.fields if isinstance(f, (CharField, TextField))]
queries = [Q(**{f.name + "__icontains": search_value}) for f in fields]
search_query = Q()
for query in queries:
search_query = search_query | query
return search_query
@api_view()
@authentication_classes([])
@permission_classes([AllowAny])
def deposit_list_datatables(request: Request) -> JsonResponse:
"""Special API view to list and filter deposits, produced responses are intended
to be consumed by datatables js framework used in deposits admin Web UI."""
table_data: Dict[str, Any] = {}
table_data["draw"] = int(request.GET.get("draw", 1))
try:
username = request.GET.get("username")
if username:
deposits = Deposit.objects.select_related("client").filter(
client__username=username
)
else:
deposits = Deposit.objects.all()
deposits_count = deposits.count()
search_value = request.GET.get("search[value]")
if search_value:
deposits = deposits.filter(_deposit_search_query(search_value))
exclude_pattern = request.GET.get("excludePattern")
if exclude_pattern:
deposits = deposits.exclude(_deposit_search_query(exclude_pattern))
column_order = request.GET.get("order[0][column]")
field_order = request.GET.get("columns[%s][name]" % column_order, "id")
order_dir = request.GET.get("order[0][dir]", "desc")
if order_dir == "desc":
field_order = "-" + field_order
deposits = deposits.order_by(field_order)
length = int(request.GET.get("length", 10))
page = int(request.GET.get("start", 0)) // length + 1
paginator = Paginator(deposits, length)
data = [
DepositSerializer(_enrich_deposit_with_metadata(d)).data
for d in paginator.page(page).object_list
]
table_data["recordsTotal"] = deposits_count
table_data["recordsFiltered"] = deposits.count()
data_list = []
for d in data:
data_dict = {
"id": d["id"],
"type": d["type"],
"external_id": d["external_id"],
"raw_metadata": d["raw_metadata"],
"reception_date": d["reception_date"],
"status": d["status"],
"status_detail": d["status_detail"],
"swhid": d["swhid"],
"swhid_context": d["swhid_context"],
}
provenance = None
raw_metadata = d["raw_metadata"]
# for meta deposit, the uri should be the url provenance
if raw_metadata and d["type"] == "meta": # metadata provenance
provenance = parse_swh_metadata_provenance(
ElementTree.fromstring(raw_metadata)
)
# For code deposits the uri is the origin
# First, trying to determine it out of the raw metadata associated with the
# deposit
elif raw_metadata and d["type"] == "code":
create_origin_url, add_to_origin_url = parse_swh_deposit_origin(
ElementTree.fromstring(raw_metadata)
)
provenance = create_origin_url or add_to_origin_url
# For code deposits, if not provided, use the origin_url
if not provenance and d["type"] == "code":
if d["origin_url"]:
provenance = d["origin_url"]
# If still not found, fallback using the swhid context
if not provenance and d["swhid_context"]:
swhid = QualifiedSWHID.from_string(d["swhid_context"])
provenance = swhid.origin
data_dict["uri"] = provenance # could be None
data_list.append(data_dict)
table_data["data"] = data_list
except Exception as exc:
sentry_sdk.capture_exception(exc)
table_data[
"error"
] = "An error occurred while retrieving the list of deposits !"
if settings.DEBUG:
table_data["error"] += "\n" + str(exc)
return JsonResponse(table_data)
# Copyright (C) 2017-2020 The Software Heritage developers
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
......@@ -10,10 +10,11 @@ from ...config import (
PRIVATE_GET_DEPOSIT_METADATA,
PRIVATE_GET_RAW_CONTENT,
PRIVATE_LIST_DEPOSITS,
PRIVATE_LIST_DEPOSITS_DATATABLES,
PRIVATE_PUT_DEPOSIT,
)
from .deposit_check import APIChecks
from .deposit_list import APIList
from .deposit_list import APIList, deposit_list_datatables
from .deposit_read import APIReadArchives, APIReadMetadata
from .deposit_update_status import APIUpdateStatus
......@@ -75,4 +76,9 @@ urlpatterns = [
name=PRIVATE_CHECK_DEPOSIT + "-nc",
),
url(r"^deposits/$", APIList.as_view(), name=PRIVATE_LIST_DEPOSITS),
url(
r"^deposits/datatables/$",
deposit_list_datatables,
name=PRIVATE_LIST_DEPOSITS_DATATABLES,
),
]
# Copyright (C) 2017-2020 The Software Heritage developers
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
......@@ -27,6 +27,7 @@ PRIVATE_CHECK_DEPOSIT = "check-deposit"
PRIVATE_PUT_DEPOSIT = "private-update"
PRIVATE_GET_DEPOSIT_METADATA = "private-read"
PRIVATE_LIST_DEPOSITS = "private-deposit-list"
PRIVATE_LIST_DEPOSITS_DATATABLES = "private-deposit-list-datatables"
ARCHIVE_KEY = "archive"
RAW_METADATA_KEY = "raw-metadata"
......
......@@ -3,11 +3,18 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.http import QueryDict
from django.test import override_settings
from django.urls import reverse_lazy as reverse
import pytest
from rest_framework import status
from swh.deposit.api.converters import convert_status_detail
from swh.deposit.config import DEPOSIT_STATUS_LOAD_SUCCESS, PRIVATE_LIST_DEPOSITS
from swh.deposit.config import (
DEPOSIT_STATUS_LOAD_SUCCESS,
PRIVATE_LIST_DEPOSITS,
PRIVATE_LIST_DEPOSITS_DATATABLES,
)
from swh.deposit.models import DEPOSIT_CODE, DEPOSIT_METADATA_ONLY, DepositClient
from swh.deposit.tests.conftest import internal_create_deposit
......@@ -35,6 +42,13 @@ STATUS_DETAIL = {
}
@pytest.fixture()
def partial_deposit_only_metadata(partial_deposit_only_metadata):
partial_deposit_only_metadata.type = DEPOSIT_METADATA_ONLY
partial_deposit_only_metadata.save()
return partial_deposit_only_metadata
def test_deposit_list(
partial_deposit_with_metadata,
partial_deposit_only_metadata,
......@@ -46,8 +60,6 @@ def test_deposit_list(
partial_deposit_with_metadata.save()
deposit1 = partial_deposit_with_metadata
deposit2 = partial_deposit_only_metadata
deposit2.type = DEPOSIT_METADATA_ONLY
deposit2.save()
deposit3 = partial_deposit
main_url = reverse(PRIVATE_LIST_DEPOSITS)
......@@ -170,3 +182,221 @@ def test_deposit_list_for_username(
id=json_response["results"][0]["client"]
)
assert deposit_client.username == user.username
@pytest.fixture()
def deposits(
partial_deposit_with_metadata,
partial_deposit_only_metadata,
partial_deposit,
completed_deposit,
complete_deposit,
):
# to cover code extracting deposit provenance from swhid_context
complete_deposit.origin_url = None
complete_deposit.save()
return [
partial_deposit_with_metadata,
partial_deposit_only_metadata,
partial_deposit,
completed_deposit,
complete_deposit,
]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_deposit_list_datatables_empty_query(
deposits,
authenticated_client,
):
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES)
deposits_dt_data = authenticated_client.get(url).json()
assert deposits_dt_data["draw"] == 1
assert deposits_dt_data["recordsTotal"] == len(deposits)
assert deposits_dt_data["recordsFiltered"] == len(deposits)
assert len(deposits_dt_data["data"]) == len(deposits)
# by default, deposits are sorted by decreasing ids
assert [d["id"] for d in deposits_dt_data["data"]] == list(
reversed(sorted([d.id for d in deposits]))
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_deposit_list_datatables_pagination(
deposits,
authenticated_client,
):
deposits_data = []
for i in range(len(deposits)):
query_params = QueryDict(mutable=True)
query_params.update(
{
"draw": i + 1,
"length": 1,
"start": i,
}
)
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES) + "?" + query_params.urlencode()
deposits_dt_data = authenticated_client.get(url).json()
deposits_data += deposits_dt_data["data"]
assert [d["id"] for d in deposits_data] == list(
reversed(sorted([d.id for d in deposits]))
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
@pytest.mark.parametrize("sort_direction", ["asc", "desc"])
def test_deposit_list_datatables_ordering(
deposits,
authenticated_client,
sort_direction,
):
deposits_date_sorted = list(sorted(deposits, key=lambda d: d.reception_date))
if sort_direction == "desc":
deposits_date_sorted = list(reversed(deposits_date_sorted))
query_params = QueryDict(mutable=True)
query_params.update(
{
"draw": 1,
"length": 10,
"start": 0,
"order[0][column]": 4,
"order[0][dir]": sort_direction,
"columns[4][name]": "reception_date",
}
)
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES) + "?" + query_params.urlencode()
deposits_dt_data = authenticated_client.get(url).json()
reception_dates = [d["reception_date"] for d in deposits_dt_data["data"]]
expected_dates = [
d.reception_date.isoformat().replace("+00:00", "Z")
for d in deposits_date_sorted
]
assert reception_dates == expected_dates
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_deposit_list_datatables_search(
deposits,
authenticated_client,
):
query_params = QueryDict(mutable=True)
query_params.update(
{
"draw": 1,
"length": 10,
"start": 0,
"search[value]": DEPOSIT_STATUS_LOAD_SUCCESS,
}
)
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES) + "?" + query_params.urlencode()
deposits_dt_data = authenticated_client.get(url).json()
deposits_load_success = [
d for d in deposits if d.status == DEPOSIT_STATUS_LOAD_SUCCESS
]
deposits_load_success = list(
reversed(sorted(deposits_load_success, key=lambda d: d.id))
)
assert deposits_load_success
assert [d.id for d in deposits_load_success] == [
d["id"] for d in deposits_dt_data["data"]
]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_deposit_list_datatables_exclude_pattern(
deposits,
authenticated_client,
):
query_params = QueryDict(mutable=True)
query_params.update(
{
"draw": 1,
"length": 10,
"start": 0,
"excludePattern": DEPOSIT_STATUS_LOAD_SUCCESS,
}
)
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES) + "?" + query_params.urlencode()
deposits_dt_data = authenticated_client.get(url).json()
deposits_load_not_success = [
d for d in deposits if d.status != DEPOSIT_STATUS_LOAD_SUCCESS
]
deposits_load_not_success = list(
reversed(sorted(deposits_load_not_success, key=lambda d: d.id))
)
assert deposits_load_not_success
assert [d.id for d in deposits_load_not_success] == [
d["id"] for d in deposits_dt_data["data"]
]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_deposit_list_datatables_username(
completed_deposit,
deposit_user,
deposit_another_user,
deposit_another_collection,
authenticated_client,
):
# create a new deposit with a user different from deposit_user,
# the one that created completed_deposit
completed_deposit_another_user = internal_create_deposit(
client=deposit_another_user,
collection=deposit_another_collection,
external_id="external-id-bar",
status=DEPOSIT_STATUS_LOAD_SUCCESS,
)
for user, deposit in (
(deposit_user, completed_deposit),
(deposit_another_user, completed_deposit_another_user),
):
query_params = QueryDict(mutable=True)
query_params["username"] = user.username
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES) + "?" + query_params.urlencode()
deposits_dt_data = authenticated_client.get(url).json()
assert len(deposits_dt_data["data"]) == 1
assert deposits_dt_data["data"][0]["id"] == deposit.id
@pytest.mark.django_db(transaction=True, reset_sequences=True)
@override_settings(DEBUG=True)
def test_deposit_list_datatables_error(
deposits,
authenticated_client,
mocker,
):
parse_swh_metadata_provenance = mocker.patch(
"swh.deposit.api.private.deposit_list.parse_swh_metadata_provenance"
)
error_message = "Error when parsing metadata"
parse_swh_metadata_provenance.side_effect = Exception(error_message)
url = reverse(PRIVATE_LIST_DEPOSITS_DATATABLES)
deposits_dt_data = authenticated_client.get(url).json()
assert "error" in deposits_dt_data
assert error_message in deposits_dt_data["error"]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment