♻️(backend) improve mimetype detection

The mimetype detection were sometimes not accurate and return a wrong
indication. For example pptx files were identified as an archive because
they are zip files. This version relies on the file extension is the
gussed mimtype is not relevant.
Guessing mimetype from file few bytes is called magic bytes
https://en.wikipedia.org/wiki/List_of_file_signatures

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to
 
 - ✨(api) modify items/search endpoint to use indexed items in Find
 - 🐛(email) avoid trying to send emails if no provider is configured
+- ♻️(backend) improve mimetype detection
 
 ### Fixed
 

src/backend/core/api/utils.py

@@ -1,5 +1,6 @@
 """Util to generate S3 authorization headers for object storage access control"""
 
+import mimetypes
 from datetime import datetime
 
 from django.conf import settings
@@ -7,6 +8,7 @@ from django.core.files.storage import default_storage
 
 import boto3
 import botocore
+import magic
 
 
 def flat_to_nested(items):
@@ -156,3 +158,58 @@ def get_item_file_head_object(item):
     return default_storage.connection.meta.client.head_object(
         Bucket=default_storage.bucket_name, Key=item.file_key
     )
+
+
+def detect_mimetype(file_buffer: bytes, filename: str | None = None) -> str:
+    """
+    Detect MIME type using multiple methods for better accuracy.
+
+    This function combines:
+    1. Magic bytes detection (python-magic) - most reliable for actual file content
+    2. File extension detection (mimetypes) - useful as fallback or for validation
+
+    Args:
+        file_buffer: The file content buffer (first bytes of the file)
+        filename: Optional filename to extract extension from
+
+    Returns:
+        str: The detected MIME type
+    """
+    # Initialize magic detector
+    mime_detector = magic.Magic(mime=True)
+
+    # Method 1: Detect from file content (magic bytes) - most reliable
+    mimetype_from_content = mime_detector.from_buffer(file_buffer)
+
+    # If we have a filename, try extension-based detection as well
+    mimetype_from_extension = None
+    if filename:
+        # Use mimetypes module to guess from extension
+        # Use guess_file_type (Python 3.13+) instead of deprecated guess_type
+        mimetype_from_extension, _ = mimetypes.guess_file_type(filename, strict=False)
+
+    # Strategy: Prefer content-based detection, but use extension if:
+    # 1. Content detection returns generic types (application/octet-stream, text/plain)
+    # 2. Content detection fails or returns None
+    # 3. Extension detection provides a more specific type
+
+    # Generic/unreliable MIME types that we should try to improve
+    generic_types = {
+        "application/octet-stream",
+        "application/zip",
+        "text/plain",
+    }
+
+    # If content detection gives us a generic type and we have extension info
+    if mimetype_from_content in generic_types and mimetype_from_extension:
+        # Use extension-based detection if it's more specific
+        if mimetype_from_extension not in generic_types:
+            return mimetype_from_extension
+
+    # If content detection failed, returned None or is a generic type, use extension if available
+    if not mimetype_from_content or mimetype_from_content in generic_types:
+        if mimetype_from_extension:
+            return mimetype_from_extension
+
+    # Default to content-based detection (most reliable)
+    return mimetype_from_content or "application/octet-stream"

src/backend/core/api/viewsets.py

@@ -20,7 +20,6 @@ from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.utils.text import slugify
 
-import magic
 import posthog
 import rest_framework as drf
 from corsheaders.middleware import (
@@ -692,7 +691,6 @@ class ItemViewSet(
                 )
             )
 
-        mime_detector = magic.Magic(mime=True)
         s3_client = default_storage.connection.meta.client
 
         head_response = s3_client.head_object(
@@ -712,7 +710,8 @@ class ItemViewSet(
                 Bucket=default_storage.bucket_name, Key=item.file_key
             )["Body"].read()
 
-        mimetype = mime_detector.from_buffer(file_head)
+        # Use improved MIME type detection combining magic bytes and file extension
+        mimetype = utils.detect_mimetype(file_head, filename=item.filename)
 
         item.upload_state = models.ItemUploadStateChoices.ANALYZING
         item.mimetype = mimetype

src/backend/core/tests/test_api_utils_detect_mimetype.py

@@ -0,0 +1,146 @@
+"""Test utils.detect_mimetype"""
+
+from core.api import utils
+
+
+def test_detect_mimetype_from_content_pdf():
+    """Test detect_mimetype detects PDF from content (magic bytes)."""
+    # PDF magic bytes: %PDF
+    pdf_content = b"%PDF-1.4\n"
+    mimetype = utils.detect_mimetype(pdf_content, filename="document.pdf")
+    assert mimetype == "application/pdf"
+
+
+def test_detect_mimetype_from_content_png():
+    """Test detect_mimetype detects PNG from content (magic bytes)."""
+    # PNG magic bytes: \x89PNG\r\n\x1a\n
+    png_content = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
+    mimetype = utils.detect_mimetype(png_content, filename="image.png")
+    assert mimetype == "image/png"
+
+
+def test_detect_mimetype_from_content_jpeg():
+    """Test detect_mimetype detects JPEG from content (magic bytes)."""
+    # JPEG magic bytes: \xff\xd8\xff
+    jpeg_content = b"\xff\xd8\xff\xe0\x00\x10JFIF"
+    mimetype = utils.detect_mimetype(jpeg_content, filename="photo.jpg")
+    assert mimetype == "image/jpeg"
+
+
+def test_detect_mimetype_from_content_text_plain():
+    """Test detect_mimetype detects plain text from content."""
+    text_content = b"This is plain text content"
+    mimetype = utils.detect_mimetype(text_content, filename="file.txt")
+    assert mimetype == "text/plain"
+
+
+def test_detect_mimetype_empty_file():
+    """Test detect_mimetype handles empty files."""
+    empty_content = b""
+    mimetype = utils.detect_mimetype(empty_content, filename="empty.txt")
+    # Empty files should be detected as application/x-empty by magic bytes
+    assert mimetype == "application/x-empty"
+
+
+def test_detect_mimetype_uses_extension_when_content_generic():
+    """Test detect_mimetype uses extension when content detection returns generic type."""
+    # Generic binary content that might be detected as application/octet-stream
+    generic_content = b"\x00\x01\x02\x03\x04\x05"
+    # But if we have a specific extension, we should use it
+    mimetype = utils.detect_mimetype(generic_content, filename="document.pdf")
+    # Should prefer extension-based detection for specific types
+    assert mimetype == "application/pdf"
+
+
+def test_detect_mimetype_uses_extension_for_json():
+    """Test detect_mimetype uses extension for JSON files."""
+    # JSON content might be detected as text/plain
+    json_content = b'{"key": "value"}'
+    mimetype = utils.detect_mimetype(json_content, filename="data.json")
+    # Should use extension to get application/json
+    assert mimetype == "application/json"
+
+
+def test_detect_mimetype_without_filename():
+    """Test detect_mimetype works without filename (content-only detection)."""
+    pdf_content = b"%PDF-1.4\n"
+    mimetype = utils.detect_mimetype(pdf_content, filename=None)
+    assert mimetype == "application/pdf"
+
+
+def test_detect_mimetype_fallback_to_extension():
+    """Test detect_mimetype falls back to extension when content detection is generic."""
+    # Content that might be detected as text/plain
+    content = b"some content"
+    mimetype = utils.detect_mimetype(content, filename="script.js")
+    # Should use extension to get JavaScript MIME type
+    # (can be text/javascript or application/javascript)
+    assert mimetype in ["text/javascript", "application/javascript"]
+
+
+def test_detect_mimetype_generic_content_no_extension():
+    """Test detect_mimetype with generic content and no extension."""
+    generic_content = b"\x00\x01\x02\x03"
+    mimetype = utils.detect_mimetype(generic_content, filename="file")
+    # Should return content-based detection (likely application/octet-stream)
+    assert mimetype in ["application/octet-stream", "application/x-empty"]
+
+
+def test_detect_mimetype_xml_file():
+    """Test detect_mimetype detects XML files."""
+    xml_content = b'<?xml version="1.0"?><root></root>'
+    mimetype = utils.detect_mimetype(xml_content, filename="data.xml")
+    # Should detect as XML (either from content or extension)
+    assert mimetype in ["application/xml", "text/xml"]
+
+
+def test_detect_mimetype_csv_file():
+    """Test detect_mimetype detects CSV files."""
+    csv_content = b"name,age\nJohn,30\nJane,25"
+    mimetype = utils.detect_mimetype(csv_content, filename="data.csv")
+    # CSV might be detected as text/plain, but extension should help
+    assert mimetype in ["text/csv", "text/plain"]
+
+
+def test_detect_mimetype_zip_file():
+    """Test detect_mimetype detects ZIP files from magic bytes."""
+    # ZIP magic bytes: PK\x03\x04
+    zip_content = b"PK\x03\x04\x14\x00\x00\x00"
+    mimetype = utils.detect_mimetype(zip_content, filename="archive.zip")
+    assert mimetype == "application/zip"
+
+
+def test_detect_mimetype_prefers_content_over_extension():
+    """Test detect_mimetype prefers content detection when both are available and specific."""
+    # PDF content but wrong extension
+    pdf_content = b"%PDF-1.4\n"
+    mimetype = utils.detect_mimetype(pdf_content, filename="document.txt")
+    # Should prefer content detection (PDF) over extension (txt)
+    assert mimetype == "application/pdf"
+
+
+def test_detect_mimetype_powerpoint_pptx():
+    """Test detect_mimetype correctly detects PowerPoint .pptx files."""
+    # .pptx files are ZIP archives, so content might be detected as application/zip or octet-stream
+    # But with the extension, it should be detected as PowerPoint MIME type
+    # Using minimal ZIP-like content that might be detected as generic
+    pptx_content = b"PK\x03\x04"  # ZIP magic bytes (PPTX is a ZIP archive)
+    mimetype = utils.detect_mimetype(pptx_content, filename="presentation.pptx")
+    # Should use extension to get PowerPoint MIME type
+    assert mimetype in [
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+    ]
+
+
+def test_detect_mimetype_powerpoint_ppt():
+    """Test detect_mimetype correctly detects PowerPoint .ppt files (older format)."""
+    # .ppt files might be detected as application/octet-stream by magic bytes
+    # But with the extension, it should be detected as PowerPoint MIME type
+    ppt_content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"  # OLE2 compound document header
+    mimetype = utils.detect_mimetype(ppt_content, filename="presentation.ppt")
+    # Should use extension to get PowerPoint MIME type
+    assert mimetype in [
+        "application/vnd.ms-powerpoint",
+        "application/mspowerpoint",
+    ]