diff --git a/docs/metrics.md b/docs/metrics.md index 651b2c36..1612a373 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -12,6 +12,8 @@ You must provide an authorization header as follows: Authorization: Api-Key ``` +You should also provide `account_type=user|organization` query parameter. + ## Enabling By default, this route is not available. You need to set the setting `METRICS_ENABLED` to `True`. @@ -69,9 +71,22 @@ The `storage_used` is, by default, the sum of file sizes created by the user, in ## Filtering -You can filter this by using `account_id` query parameter like so: +You can filter this by using `account_id_key` and `account_id_value` query parameter like so: -`GET /external_api/v1.0/metrics/usage/?account_id=` +- `account_type` | `user` or `organization` +- `account_id_key` | The key to filter on (`sub` or `email` for `account_type=user`). Required if `account_type=organization`, it could be any oidc stored claim. +- `account_id_value` | The value to filter on. Required if `account_type=organization`. +- `account_email` | This will soon be migrated to \_key \_value pattern, but still needed. + +Examples: + +``` +GET /external_api/v1.0/metrics/usage/?account_type=user +GET /external_api/v1.0/metrics/usage/?account_type=user&account_id_key=sub&account_id_value= +GET /external_api/v1.0/metrics/usage/?account_type=user&account_email= +GET /external_api/v1.0/metrics/usage/?account_type=organization # Forbidden +GET /external_api/v1.0/metrics/usage/?account_type=organization&account_id_key=siret&account_id_value= +``` ## How to customize the storage used computation? @@ -83,7 +98,7 @@ You can see the default implementation `CreatorStorageComputeBackend` as an exam For aggregation purposes, you might want to expose some OIDC claims from this route that your external tool will use. -For instance, with the ANCT, the implementation needs to get access to the `siret` claim. +For instance, with the deploy center, the implementation needs to get access to the `siret` claim. Simply customize the setting `METRICS_USER_CLAIMS_EXPOSED=your_claim1,your_claim2` and the response will look like this: @@ -111,3 +126,34 @@ Simply customize the setting `METRICS_USER_CLAIMS_EXPOSED=your_claim1,your_claim ``` 🚨 Make sure the claims you want to expose are stored by using the `OIDC_STORE_CLAIMS` setting as well. Otherwise, it will not work. 🚨 + +## Organization metrics + +It is possibe to fetch usage metric grouped by organization, do to so, provide `account_type=organization`, when using account_type organization it is required that you also provide `account_id_key=siret` and `account_id_value=1234...`. + +It will sum the storage_used by user that have the same "claims.siret". + +> You should mind to add `siret` into OIDC_STORE_CLAIMS so the siret are stored on user records. + +Example: + +``` +GET /external_api/v1.0/metrics/usage?account_type=organization&account_id_key=siret&account_id_value=12345678900001 +{ + "count": 123, + "next": None, + "previous": None, + "results": [ + { + "account": { + "type": "organization", + }, + "siret": "12345678900001" + "metrics": { + "storage_used": 10000000, + }, + }, + ... + ], +} +``` diff --git a/src/backend/core/api/filters.py b/src/backend/core/api/filters.py index ad810b79..af3fb200 100644 --- a/src/backend/core/api/filters.py +++ b/src/backend/core/api/filters.py @@ -167,3 +167,68 @@ class ListItemFilter(ItemFilter): return queryset return queryset.filter(is_favorite=bool(value)) + + +class UsageMetricAccountTypeChoices(TextChoices): + """Choices for the usage metrics `account_type` query param.""" + + USER = "user", _("User") + ORGANIZATION = "organization", _("Organization") + + +class UsageMetricAccountIdKeyChoices(TextChoices): + """Allowed keys for filtering users by account id in the usage metrics endpoint.""" + + SUB = "sub", _("Sub") + EMAIL = "email", _("Email") + + +class BaseUsageMetricFilter(django_filters.FilterSet): + """Shared `account_id_key`/`account_id_value` handling for usage metrics filters. + + Subclasses declare their own `account_id_key` filter (with the right validation + rules and `required` flag) and override `ACCOUNT_ID_LOOKUP` to point at the field + path used to filter the queryset. + """ + + ACCOUNT_ID_LOOKUP = "{key}" + + account_id_value = django_filters.CharFilter(method="filter_noop") + + # pylint: disable=unused-argument + def filter_account_id_key(self, queryset, name, value): + """Apply the account_id_key/account_id_value pair as a single filter.""" + account_id_value = self.data.get("account_id_value") + if not account_id_value: + return queryset + lookup = self.ACCOUNT_ID_LOOKUP.format(key=value) + return queryset.filter(**{lookup: account_id_value}) + + # pylint: disable=unused-argument + def filter_noop(self, queryset, name, value): + """No-op: `account_id_value` is consumed by `filter_account_id`.""" + return queryset + + +class UsageMetricFilter(BaseUsageMetricFilter): + """Filter for the usage metrics endpoint (user listing).""" + + account_id_key = django_filters.ChoiceFilter( + choices=UsageMetricAccountIdKeyChoices.choices, + method="filter_account_id_key", + ) + account_email = django_filters.CharFilter(field_name="email") + + +class OrganizationUsageMetricFilter(BaseUsageMetricFilter): + """Filter for the organization variant of the usage metrics endpoint. + + Both `account_id_key` and `account_id_value` are required, the key is an + arbitrary OIDC claim name, and the lookup goes through the User's `claims` + JSON field. + """ + + ACCOUNT_ID_LOOKUP = "claims__{key}" + + account_id_key = django_filters.CharFilter(method="filter_account_id_key", required=True) + account_id_value = django_filters.CharFilter(method="filter_noop", required=True) diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 371ecf46..dcd52e59 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -65,8 +65,8 @@ class UserLightSerializer(UserSerializer): # pylint: disable=abstract-method -class UsageMetricSerializer(serializers.BaseSerializer): - """Serialize usage metrics.""" +class UserUsageMetricSerializer(serializers.BaseSerializer): + """Serialize usage metrics for a single user.""" def to_representation(self, instance): """Return the usage metric.""" @@ -78,7 +78,9 @@ class UsageMetricSerializer(serializers.BaseSerializer): "email": instance.email, }, "metrics": { - "storage_used": storage_compute_backend.compute_storage_used(instance), + "storage_used": storage_compute_backend.compute_storage_used( + models.User.objects.filter(pk=instance.pk) + ), }, } for claim in settings.METRICS_USER_CLAIMS_EXPOSED: @@ -86,6 +88,22 @@ class UsageMetricSerializer(serializers.BaseSerializer): return output +class OrganizationUsageMetricSerializer(serializers.Serializer): + """Serialize aggregated usage metrics for an organization.""" + + account_id_key = serializers.CharField() + account_id_value = serializers.CharField() + total_storage = serializers.IntegerField() + + def to_representation(self, instance): + """Return the organization usage metric.""" + return { + "account": {"type": "organization"}, + instance["account_id_key"]: instance["account_id_value"], + "metrics": {"storage_used": instance["total_storage"]}, + } + + class ItemLightSerializer(serializers.ModelSerializer): """Minimal item serializer for nesting in item accesses.""" diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index b1e9766b..e1216ca8 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -50,13 +50,22 @@ from core.services.search_indexers import ( get_file_indexer, get_visited_items_ids_of, ) +from core.storage import get_storage_compute_backend from core.tasks.item import duplicate_file, process_item_purge, rename_file from core.utils.analytics import posthog_capture from wopi.services import access as access_service from wopi.utils import compute_wopi_launch_url, get_wopi_client_config from . import permissions, serializers, utils -from .filters import ItemFilter, ItemOrdering, ListItemFilter, SearchItemFilter +from .filters import ( + ItemFilter, + ItemOrdering, + ListItemFilter, + OrganizationUsageMetricFilter, + SearchItemFilter, + UsageMetricAccountTypeChoices, + UsageMetricFilter, +) logger = logging.getLogger(__name__) @@ -2216,19 +2225,54 @@ class UsageMetricViewset(drf.mixins.ListModelMixin, viewsets.GenericViewSet): permission_classes = [HasAPIKey] queryset = models.User.objects.all().filter(is_active=True) - serializer_class = serializers.UsageMetricSerializer + serializer_class = serializers.UserUsageMetricSerializer pagination_class = Pagination def get_queryset(self): - """ - Return the queryset applying the filters from the query params. - """ - queryset = self.queryset + """Return the queryset filtered through `UsageMetricFilter`.""" + filterset = UsageMetricFilter( + self.request.GET, queryset=self.queryset, request=self.request + ) + if not filterset.is_valid(): + raise drf.exceptions.ValidationError(filterset.errors) + return filterset.filter_queryset(self.queryset) - if self.request.query_params.get("account_id"): - queryset = queryset.filter(sub=self.request.query_params.get("account_id")) + def list(self, request, *args, **kwargs): + """Handle listing with account_type branching.""" + account_type = request.query_params.get("account_type", UsageMetricAccountTypeChoices.USER) - return queryset + if account_type == UsageMetricAccountTypeChoices.ORGANIZATION: + return self._list_organization(request) + + return super().list(request, *args, **kwargs) + + def _list_organization(self, request): + """Aggregate storage metrics across users of an organization.""" + base_qs = models.User.objects.filter(is_active=True) + filterset = OrganizationUsageMetricFilter(request.GET, queryset=base_qs, request=request) + if not filterset.is_valid(): + raise drf.exceptions.ValidationError(filterset.errors) + users = filterset.filter_queryset(base_qs) + + storage_backend = get_storage_compute_backend() + total_storage = storage_backend.compute_storage_used(users) + + serializer = serializers.OrganizationUsageMetricSerializer( + { + "account_id_key": filterset.form.cleaned_data["account_id_key"], + "account_id_value": filterset.form.cleaned_data["account_id_value"], + "total_storage": total_storage, + } + ) + + return drf.response.Response( + { + "count": 1, + "next": None, + "previous": None, + "results": [serializer.data], + } + ) class EntitlementsViewset(viewsets.ViewSet): @@ -2247,4 +2291,5 @@ class EntitlementsViewset(viewsets.ViewSet): method = getattr(entitlements_backend, method_name) if callable(method): entitlements[method_name] = method(request.user) + entitlements["context"] = entitlements_backend.get_context(request.user) return drf.response.Response(entitlements) diff --git a/src/backend/core/tests/external_api/metrics/test_external_api_usage_metrics.py b/src/backend/core/tests/external_api/metrics/test_external_api_usage_metrics.py index 412ae61f..8381a718 100644 --- a/src/backend/core/tests/external_api/metrics/test_external_api_usage_metrics.py +++ b/src/backend/core/tests/external_api/metrics/test_external_api_usage_metrics.py @@ -134,7 +134,57 @@ def test_usage_metrics_list_filter_by_account_id(api_key): factories.ItemFactory(creator=user1, size=100) response = client.get( - f"/external_api/v1.0/metrics/usage/?account_id={user1.sub}", + f"/external_api/v1.0/metrics/usage/?account_id_key=sub&account_id_value={user1.sub}", + HTTP_AUTHORIZATION=f"Api-Key {api_key}", + ) + assert response.status_code == 200 + assert response.json() == { + "count": 1, + "next": None, + "previous": None, + "results": [ + { + "account": { + "type": "user", + "id": str(user1.sub), + "email": user1.email, + }, + "metrics": { + "storage_used": 100, + }, + }, + ], + } + + +@override_settings(METRICS_ENABLED=True) +def test_usage_metrics_list_invalid_account_id_key(api_key): + """An unsupported account_id_key should be rejected with a 400.""" + client = APIClient() + + response = client.get( + "/external_api/v1.0/metrics/usage/?account_id_key=bogus&account_id_value=42", + HTTP_AUTHORIZATION=f"Api-Key {api_key}", + ) + assert response.status_code == 400 + body = response.json() + assert body["type"] == "validation_error" + assert any(error["attr"] == "account_id_key" for error in body["errors"]) + + +@override_settings(METRICS_ENABLED=True) +def test_usage_metrics_list_filter_by_account_email(api_key): + """ + API keys should be allowed to list usage metrics for a specific account. + """ + client = APIClient() + user1 = factories.UserFactory() + factories.UserFactory() + + factories.ItemFactory(creator=user1, size=100) + + response = client.get( + f"/external_api/v1.0/metrics/usage/?account_email={user1.email}", HTTP_AUTHORIZATION=f"Api-Key {api_key}", ) assert response.status_code == 200 @@ -207,3 +257,84 @@ def test_usage_metrics_exposed_claims(api_key): }, ], } + + +@override_settings(METRICS_ENABLED=True) +def test_usage_metrics_organization_type(api_key): + """ + Organization metrics should aggregate storage across users + sharing the same claim value. + """ + client = APIClient() + siret = "12345678901234" + user1 = factories.UserFactory(claims={"siret": siret}) + user2 = factories.UserFactory(claims={"siret": siret}) + factories.UserFactory(claims={"siret": "99999999999999"}) + + factories.ItemFactory(creator=user1, size=100) + factories.ItemFactory(creator=user2, size=250) + + response = client.get( + "/external_api/v1.0/metrics/usage/" + f"?account_type=organization&account_id_key=siret" + f"&account_id_value={siret}", + HTTP_AUTHORIZATION=f"Api-Key {api_key}", + ) + assert response.status_code == 200 + assert response.json() == { + "count": 1, + "next": None, + "previous": None, + "results": [ + { + "account": {"type": "organization"}, + "siret": siret, + "metrics": {"storage_used": 350}, + }, + ], + } + + +@override_settings(METRICS_ENABLED=True) +def test_usage_metrics_organization_type_missing_params(api_key): + """ + Organization metrics should return 400 when account_id_key + or account_id_value is missing. + """ + client = APIClient() + + response = client.get( + "/external_api/v1.0/metrics/usage/?account_type=organization", + HTTP_AUTHORIZATION=f"Api-Key {api_key}", + ) + assert response.status_code == 400 + + +@override_settings(METRICS_ENABLED=True) +def test_usage_metrics_organization_type_no_matching_users(api_key): + """ + Organization metrics should return storage_used=0 when no users + match the given claim. + """ + client = APIClient() + factories.UserFactory(claims={"siret": "11111111111111"}) + + response = client.get( + "/external_api/v1.0/metrics/usage/" + "?account_type=organization&account_id_key=siret" + "&account_id_value=00000000000000", + HTTP_AUTHORIZATION=f"Api-Key {api_key}", + ) + assert response.status_code == 200 + assert response.json() == { + "count": 1, + "next": None, + "previous": None, + "results": [ + { + "account": {"type": "organization"}, + "siret": "00000000000000", + "metrics": {"storage_used": 0}, + }, + ], + }