mirror of
https://github.com/suitenumerique/drive.git
synced 2026-08-17 20:15:40 +02:00
✨(backend) add organization metrics to usage API
Add account_type=organization support to the usage metrics endpoint. When requested, the API filters users by a shared OIDC claim (account_id_key/account_id_value) and returns aggregated storage across all matching users. Add handling of account_id_key/account_id_value pair, allowing filtering on allowed parameters in a more generic way. At the moment `account_email` filter is still needed. Also include the context of the entitlement in the /entitlements response.
This commit is contained in:
Nathan Vasse
+49
-3
@@ -12,6 +12,8 @@ You must provide an authorization header as follows:
|
||||
Authorization: Api-Key <key>
|
||||
```
|
||||
|
||||
You should also provide `account_type=user|organization` query parameter.
|
||||
|
||||
## Enabling
|
||||
|
||||
By default, this route is not available. You need to set the setting `METRICS_ENABLED` to `True`.
|
||||
@@ -69,9 +71,22 @@ The `storage_used` is, by default, the sum of file sizes created by the user, in
|
||||
|
||||
## Filtering
|
||||
|
||||
You can filter this by using `account_id` query parameter like so:
|
||||
You can filter this by using `account_id_key` and `account_id_value` query parameter like so:
|
||||
|
||||
`GET /external_api/v1.0/metrics/usage/?account_id=<uuid>`
|
||||
- `account_type` | `user` or `organization`
|
||||
- `account_id_key` | The key to filter on (`sub` or `email` for `account_type=user`). Required if `account_type=organization`, it could be any oidc stored claim.
|
||||
- `account_id_value` | The value to filter on. Required if `account_type=organization`.
|
||||
- `account_email` | This will soon be migrated to \_key \_value pattern, but still needed.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
GET /external_api/v1.0/metrics/usage/?account_type=user
|
||||
GET /external_api/v1.0/metrics/usage/?account_type=user&account_id_key=sub&account_id_value=<sub value>
|
||||
GET /external_api/v1.0/metrics/usage/?account_type=user&account_email=<email>
|
||||
GET /external_api/v1.0/metrics/usage/?account_type=organization # Forbidden
|
||||
GET /external_api/v1.0/metrics/usage/?account_type=organization&account_id_key=siret&account_id_value=<siret value>
|
||||
```
|
||||
|
||||
## How to customize the storage used computation?
|
||||
|
||||
@@ -83,7 +98,7 @@ You can see the default implementation `CreatorStorageComputeBackend` as an exam
|
||||
|
||||
For aggregation purposes, you might want to expose some OIDC claims from this route that your external tool will use.
|
||||
|
||||
For instance, with the ANCT, the implementation needs to get access to the `siret` claim.
|
||||
For instance, with the deploy center, the implementation needs to get access to the `siret` claim.
|
||||
|
||||
Simply customize the setting `METRICS_USER_CLAIMS_EXPOSED=your_claim1,your_claim2` and the response will look like this:
|
||||
|
||||
@@ -111,3 +126,34 @@ Simply customize the setting `METRICS_USER_CLAIMS_EXPOSED=your_claim1,your_claim
|
||||
```
|
||||
|
||||
🚨 Make sure the claims you want to expose are stored by using the `OIDC_STORE_CLAIMS` setting as well. Otherwise, it will not work. 🚨
|
||||
|
||||
## Organization metrics
|
||||
|
||||
It is possibe to fetch usage metric grouped by organization, do to so, provide `account_type=organization`, when using account_type organization it is required that you also provide `account_id_key=siret` and `account_id_value=1234...`.
|
||||
|
||||
It will sum the storage_used by user that have the same "claims.siret".
|
||||
|
||||
> You should mind to add `siret` into OIDC_STORE_CLAIMS so the siret are stored on user records.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
GET /external_api/v1.0/metrics/usage?account_type=organization&account_id_key=siret&account_id_value=12345678900001
|
||||
{
|
||||
"count": 123,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [
|
||||
{
|
||||
"account": {
|
||||
"type": "organization",
|
||||
},
|
||||
"siret": "12345678900001"
|
||||
"metrics": {
|
||||
"storage_used": 10000000,
|
||||
},
|
||||
},
|
||||
...
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
@@ -167,3 +167,68 @@ class ListItemFilter(ItemFilter):
|
||||
return queryset
|
||||
|
||||
return queryset.filter(is_favorite=bool(value))
|
||||
|
||||
|
||||
class UsageMetricAccountTypeChoices(TextChoices):
|
||||
"""Choices for the usage metrics `account_type` query param."""
|
||||
|
||||
USER = "user", _("User")
|
||||
ORGANIZATION = "organization", _("Organization")
|
||||
|
||||
|
||||
class UsageMetricAccountIdKeyChoices(TextChoices):
|
||||
"""Allowed keys for filtering users by account id in the usage metrics endpoint."""
|
||||
|
||||
SUB = "sub", _("Sub")
|
||||
EMAIL = "email", _("Email")
|
||||
|
||||
|
||||
class BaseUsageMetricFilter(django_filters.FilterSet):
|
||||
"""Shared `account_id_key`/`account_id_value` handling for usage metrics filters.
|
||||
|
||||
Subclasses declare their own `account_id_key` filter (with the right validation
|
||||
rules and `required` flag) and override `ACCOUNT_ID_LOOKUP` to point at the field
|
||||
path used to filter the queryset.
|
||||
"""
|
||||
|
||||
ACCOUNT_ID_LOOKUP = "{key}"
|
||||
|
||||
account_id_value = django_filters.CharFilter(method="filter_noop")
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def filter_account_id_key(self, queryset, name, value):
|
||||
"""Apply the account_id_key/account_id_value pair as a single filter."""
|
||||
account_id_value = self.data.get("account_id_value")
|
||||
if not account_id_value:
|
||||
return queryset
|
||||
lookup = self.ACCOUNT_ID_LOOKUP.format(key=value)
|
||||
return queryset.filter(**{lookup: account_id_value})
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def filter_noop(self, queryset, name, value):
|
||||
"""No-op: `account_id_value` is consumed by `filter_account_id`."""
|
||||
return queryset
|
||||
|
||||
|
||||
class UsageMetricFilter(BaseUsageMetricFilter):
|
||||
"""Filter for the usage metrics endpoint (user listing)."""
|
||||
|
||||
account_id_key = django_filters.ChoiceFilter(
|
||||
choices=UsageMetricAccountIdKeyChoices.choices,
|
||||
method="filter_account_id_key",
|
||||
)
|
||||
account_email = django_filters.CharFilter(field_name="email")
|
||||
|
||||
|
||||
class OrganizationUsageMetricFilter(BaseUsageMetricFilter):
|
||||
"""Filter for the organization variant of the usage metrics endpoint.
|
||||
|
||||
Both `account_id_key` and `account_id_value` are required, the key is an
|
||||
arbitrary OIDC claim name, and the lookup goes through the User's `claims`
|
||||
JSON field.
|
||||
"""
|
||||
|
||||
ACCOUNT_ID_LOOKUP = "claims__{key}"
|
||||
|
||||
account_id_key = django_filters.CharFilter(method="filter_account_id_key", required=True)
|
||||
account_id_value = django_filters.CharFilter(method="filter_noop", required=True)
|
||||
|
||||
@@ -65,8 +65,8 @@ class UserLightSerializer(UserSerializer):
|
||||
|
||||
|
||||
# pylint: disable=abstract-method
|
||||
class UsageMetricSerializer(serializers.BaseSerializer):
|
||||
"""Serialize usage metrics."""
|
||||
class UserUsageMetricSerializer(serializers.BaseSerializer):
|
||||
"""Serialize usage metrics for a single user."""
|
||||
|
||||
def to_representation(self, instance):
|
||||
"""Return the usage metric."""
|
||||
@@ -78,7 +78,9 @@ class UsageMetricSerializer(serializers.BaseSerializer):
|
||||
"email": instance.email,
|
||||
},
|
||||
"metrics": {
|
||||
"storage_used": storage_compute_backend.compute_storage_used(instance),
|
||||
"storage_used": storage_compute_backend.compute_storage_used(
|
||||
models.User.objects.filter(pk=instance.pk)
|
||||
),
|
||||
},
|
||||
}
|
||||
for claim in settings.METRICS_USER_CLAIMS_EXPOSED:
|
||||
@@ -86,6 +88,22 @@ class UsageMetricSerializer(serializers.BaseSerializer):
|
||||
return output
|
||||
|
||||
|
||||
class OrganizationUsageMetricSerializer(serializers.Serializer):
|
||||
"""Serialize aggregated usage metrics for an organization."""
|
||||
|
||||
account_id_key = serializers.CharField()
|
||||
account_id_value = serializers.CharField()
|
||||
total_storage = serializers.IntegerField()
|
||||
|
||||
def to_representation(self, instance):
|
||||
"""Return the organization usage metric."""
|
||||
return {
|
||||
"account": {"type": "organization"},
|
||||
instance["account_id_key"]: instance["account_id_value"],
|
||||
"metrics": {"storage_used": instance["total_storage"]},
|
||||
}
|
||||
|
||||
|
||||
class ItemLightSerializer(serializers.ModelSerializer):
|
||||
"""Minimal item serializer for nesting in item accesses."""
|
||||
|
||||
|
||||
@@ -50,13 +50,22 @@ from core.services.search_indexers import (
|
||||
get_file_indexer,
|
||||
get_visited_items_ids_of,
|
||||
)
|
||||
from core.storage import get_storage_compute_backend
|
||||
from core.tasks.item import duplicate_file, process_item_purge, rename_file
|
||||
from core.utils.analytics import posthog_capture
|
||||
from wopi.services import access as access_service
|
||||
from wopi.utils import compute_wopi_launch_url, get_wopi_client_config
|
||||
|
||||
from . import permissions, serializers, utils
|
||||
from .filters import ItemFilter, ItemOrdering, ListItemFilter, SearchItemFilter
|
||||
from .filters import (
|
||||
ItemFilter,
|
||||
ItemOrdering,
|
||||
ListItemFilter,
|
||||
OrganizationUsageMetricFilter,
|
||||
SearchItemFilter,
|
||||
UsageMetricAccountTypeChoices,
|
||||
UsageMetricFilter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -2216,19 +2225,54 @@ class UsageMetricViewset(drf.mixins.ListModelMixin, viewsets.GenericViewSet):
|
||||
|
||||
permission_classes = [HasAPIKey]
|
||||
queryset = models.User.objects.all().filter(is_active=True)
|
||||
serializer_class = serializers.UsageMetricSerializer
|
||||
serializer_class = serializers.UserUsageMetricSerializer
|
||||
pagination_class = Pagination
|
||||
|
||||
def get_queryset(self):
|
||||
"""
|
||||
Return the queryset applying the filters from the query params.
|
||||
"""
|
||||
queryset = self.queryset
|
||||
"""Return the queryset filtered through `UsageMetricFilter`."""
|
||||
filterset = UsageMetricFilter(
|
||||
self.request.GET, queryset=self.queryset, request=self.request
|
||||
)
|
||||
if not filterset.is_valid():
|
||||
raise drf.exceptions.ValidationError(filterset.errors)
|
||||
return filterset.filter_queryset(self.queryset)
|
||||
|
||||
if self.request.query_params.get("account_id"):
|
||||
queryset = queryset.filter(sub=self.request.query_params.get("account_id"))
|
||||
def list(self, request, *args, **kwargs):
|
||||
"""Handle listing with account_type branching."""
|
||||
account_type = request.query_params.get("account_type", UsageMetricAccountTypeChoices.USER)
|
||||
|
||||
return queryset
|
||||
if account_type == UsageMetricAccountTypeChoices.ORGANIZATION:
|
||||
return self._list_organization(request)
|
||||
|
||||
return super().list(request, *args, **kwargs)
|
||||
|
||||
def _list_organization(self, request):
|
||||
"""Aggregate storage metrics across users of an organization."""
|
||||
base_qs = models.User.objects.filter(is_active=True)
|
||||
filterset = OrganizationUsageMetricFilter(request.GET, queryset=base_qs, request=request)
|
||||
if not filterset.is_valid():
|
||||
raise drf.exceptions.ValidationError(filterset.errors)
|
||||
users = filterset.filter_queryset(base_qs)
|
||||
|
||||
storage_backend = get_storage_compute_backend()
|
||||
total_storage = storage_backend.compute_storage_used(users)
|
||||
|
||||
serializer = serializers.OrganizationUsageMetricSerializer(
|
||||
{
|
||||
"account_id_key": filterset.form.cleaned_data["account_id_key"],
|
||||
"account_id_value": filterset.form.cleaned_data["account_id_value"],
|
||||
"total_storage": total_storage,
|
||||
}
|
||||
)
|
||||
|
||||
return drf.response.Response(
|
||||
{
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [serializer.data],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class EntitlementsViewset(viewsets.ViewSet):
|
||||
@@ -2247,4 +2291,5 @@ class EntitlementsViewset(viewsets.ViewSet):
|
||||
method = getattr(entitlements_backend, method_name)
|
||||
if callable(method):
|
||||
entitlements[method_name] = method(request.user)
|
||||
entitlements["context"] = entitlements_backend.get_context(request.user)
|
||||
return drf.response.Response(entitlements)
|
||||
|
||||
@@ -134,7 +134,57 @@ def test_usage_metrics_list_filter_by_account_id(api_key):
|
||||
factories.ItemFactory(creator=user1, size=100)
|
||||
|
||||
response = client.get(
|
||||
f"/external_api/v1.0/metrics/usage/?account_id={user1.sub}",
|
||||
f"/external_api/v1.0/metrics/usage/?account_id_key=sub&account_id_value={user1.sub}",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [
|
||||
{
|
||||
"account": {
|
||||
"type": "user",
|
||||
"id": str(user1.sub),
|
||||
"email": user1.email,
|
||||
},
|
||||
"metrics": {
|
||||
"storage_used": 100,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@override_settings(METRICS_ENABLED=True)
|
||||
def test_usage_metrics_list_invalid_account_id_key(api_key):
|
||||
"""An unsupported account_id_key should be rejected with a 400."""
|
||||
client = APIClient()
|
||||
|
||||
response = client.get(
|
||||
"/external_api/v1.0/metrics/usage/?account_id_key=bogus&account_id_value=42",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
body = response.json()
|
||||
assert body["type"] == "validation_error"
|
||||
assert any(error["attr"] == "account_id_key" for error in body["errors"])
|
||||
|
||||
|
||||
@override_settings(METRICS_ENABLED=True)
|
||||
def test_usage_metrics_list_filter_by_account_email(api_key):
|
||||
"""
|
||||
API keys should be allowed to list usage metrics for a specific account.
|
||||
"""
|
||||
client = APIClient()
|
||||
user1 = factories.UserFactory()
|
||||
factories.UserFactory()
|
||||
|
||||
factories.ItemFactory(creator=user1, size=100)
|
||||
|
||||
response = client.get(
|
||||
f"/external_api/v1.0/metrics/usage/?account_email={user1.email}",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
@@ -207,3 +257,84 @@ def test_usage_metrics_exposed_claims(api_key):
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@override_settings(METRICS_ENABLED=True)
|
||||
def test_usage_metrics_organization_type(api_key):
|
||||
"""
|
||||
Organization metrics should aggregate storage across users
|
||||
sharing the same claim value.
|
||||
"""
|
||||
client = APIClient()
|
||||
siret = "12345678901234"
|
||||
user1 = factories.UserFactory(claims={"siret": siret})
|
||||
user2 = factories.UserFactory(claims={"siret": siret})
|
||||
factories.UserFactory(claims={"siret": "99999999999999"})
|
||||
|
||||
factories.ItemFactory(creator=user1, size=100)
|
||||
factories.ItemFactory(creator=user2, size=250)
|
||||
|
||||
response = client.get(
|
||||
"/external_api/v1.0/metrics/usage/"
|
||||
f"?account_type=organization&account_id_key=siret"
|
||||
f"&account_id_value={siret}",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [
|
||||
{
|
||||
"account": {"type": "organization"},
|
||||
"siret": siret,
|
||||
"metrics": {"storage_used": 350},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@override_settings(METRICS_ENABLED=True)
|
||||
def test_usage_metrics_organization_type_missing_params(api_key):
|
||||
"""
|
||||
Organization metrics should return 400 when account_id_key
|
||||
or account_id_value is missing.
|
||||
"""
|
||||
client = APIClient()
|
||||
|
||||
response = client.get(
|
||||
"/external_api/v1.0/metrics/usage/?account_type=organization",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@override_settings(METRICS_ENABLED=True)
|
||||
def test_usage_metrics_organization_type_no_matching_users(api_key):
|
||||
"""
|
||||
Organization metrics should return storage_used=0 when no users
|
||||
match the given claim.
|
||||
"""
|
||||
client = APIClient()
|
||||
factories.UserFactory(claims={"siret": "11111111111111"})
|
||||
|
||||
response = client.get(
|
||||
"/external_api/v1.0/metrics/usage/"
|
||||
"?account_type=organization&account_id_key=siret"
|
||||
"&account_id_value=00000000000000",
|
||||
HTTP_AUTHORIZATION=f"Api-Key {api_key}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [
|
||||
{
|
||||
"account": {"type": "organization"},
|
||||
"siret": "00000000000000",
|
||||
"metrics": {"storage_used": 0},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user