mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix(attachments): normalize EXIF orientation so thumbnails and models see upright images
Phone photos store landscape pixels plus an EXIF orientation tag. Browsers honour the tag for <img>, but Pillow (our thumbnails) and many vision-model image decoders do not — so the thumbnail rendered rotated AND the model literally perceived the photo sideways (noticed earlier as model "hallucinations", before thumbnails made the rotation visible). Normalize on read, at both surfaces: - new core/images.normalize_image_orientation: bakes the rotation into the pixels and re-encodes (preserving format); images with no / identity orientation pass through untouched (pristine original, no per-send cost). - make_thumbnail applies exif_transpose — after the decompression-bomb pixel gate, which now also covers the transpose decode. - attachment_to_content_part runs image bytes through the normalizer before base64, so the primary model and the perception model both get upright pixels. Because normalization is on read (not at upload), it fixes already-stored uploads too.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
"""Tests for EXIF-orientation normalisation (turnstone.core.images)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.images import normalize_image_orientation
|
||||
|
||||
Image = pytest.importorskip("PIL.Image")
|
||||
|
||||
_ORIENTATION_TAG = 0x0112 # EXIF orientation (standard tag id)
|
||||
|
||||
|
||||
def _oriented_jpeg(orientation: int, size: tuple[int, int] = (4, 2)) -> bytes:
|
||||
img = Image.new("RGB", size, "red")
|
||||
exif = img.getexif()
|
||||
exif[_ORIENTATION_TAG] = orientation
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="JPEG", exif=exif)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def test_applies_rotation_and_strips_tag() -> None:
|
||||
# Orientation 6 = "rotate 90° for display": a 4×2 landscape becomes 2×4.
|
||||
data = _oriented_jpeg(6, size=(4, 2))
|
||||
out = normalize_image_orientation(data)
|
||||
assert out != data, "a rotated image must be re-encoded upright"
|
||||
img = Image.open(BytesIO(out))
|
||||
assert img.size == (2, 4), "the 90° rotation must be baked into the pixels"
|
||||
assert img.getexif().get(_ORIENTATION_TAG) in (None, 1), "the orientation tag must be cleared"
|
||||
|
||||
|
||||
def test_passthrough_when_upright() -> None:
|
||||
data = _oriented_jpeg(1, size=(4, 2))
|
||||
assert normalize_image_orientation(data) == data, "identity orientation must not re-encode"
|
||||
|
||||
|
||||
def test_passthrough_when_no_exif() -> None:
|
||||
buf = BytesIO()
|
||||
Image.new("RGB", (3, 3), "blue").save(buf, format="PNG")
|
||||
data = buf.getvalue()
|
||||
assert normalize_image_orientation(data) == data, "a tag-less image must pass through verbatim"
|
||||
|
||||
|
||||
def test_never_raises_on_garbage() -> None:
|
||||
assert normalize_image_orientation(b"not an image") == b"not an image"
|
||||
assert normalize_image_orientation(b"") == b""
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.thumbnails import make_thumbnail
|
||||
@@ -42,6 +44,18 @@ class TestMakeThumbnail:
|
||||
out = make_thumbnail(PNG_1x1, "image")
|
||||
assert out is not None and out[:8] == _PNG_MAGIC
|
||||
|
||||
def test_image_thumbnail_honours_exif_orientation(self) -> None:
|
||||
pil = pytest.importorskip("PIL.Image")
|
||||
src = pil.new("RGB", (40, 20), "red") # landscape source
|
||||
exif = src.getexif()
|
||||
exif[0x0112] = 6 # "rotate 90° for display" → the thumbnail should be portrait
|
||||
buf = BytesIO()
|
||||
src.save(buf, format="JPEG", exif=exif)
|
||||
out = make_thumbnail(buf.getvalue(), "image")
|
||||
assert out is not None
|
||||
thumb = pil.open(BytesIO(out))
|
||||
assert thumb.height > thumb.width, "thumbnail must reflect the applied EXIF rotation"
|
||||
|
||||
def test_pdf_thumbnail_is_png(self) -> None:
|
||||
out = make_thumbnail(_minimal_pdf(), "pdf")
|
||||
assert out is not None and out[:8] == _PNG_MAGIC
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Image-pixel utilities shared across the thumbnail, wire, and perception paths.
|
||||
|
||||
Currently: EXIF-orientation normalisation. Kept separate from
|
||||
:mod:`turnstone.core.thumbnails` (which downscales for the UI) — this operates on
|
||||
full-resolution bytes at the read/wire boundary so every consumer of an
|
||||
attachment sees the same upright pixels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from turnstone.core.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# EXIF tag 0x0112 (274) — image orientation (1 = upright; 2-8 = flips/rotations).
|
||||
_EXIF_ORIENTATION_TAG = 0x0112
|
||||
|
||||
# Mirror turnstone.core.thumbnails: bound decoded pixels so a small compressed
|
||||
# file that expands to an enormous bitmap can't OOM the node during re-encode.
|
||||
_MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def normalize_image_orientation(data: bytes) -> bytes:
|
||||
"""Bake an image's EXIF orientation into its pixels; return re-encoded bytes.
|
||||
|
||||
Images with no orientation tag (or an identity orientation) are returned
|
||||
UNCHANGED — no decode/re-encode, so the pristine original is preserved and
|
||||
there is no per-send cost in the common case. Never raises: any failure
|
||||
(Pillow missing, decode error, oversized) returns the original bytes.
|
||||
|
||||
Why this exists: a phone photo stores landscape pixels plus an orientation
|
||||
tag. Browsers honour the tag for ``<img>``, but Pillow (our thumbnails) and
|
||||
many vision-model image decoders do NOT — so the model literally perceives
|
||||
the photo rotated. Normalising at the read/wire boundary makes every
|
||||
consumer (browser, thumbnail, model) see the same upright image.
|
||||
"""
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
except ImportError: # pragma: no cover - declared dependency; defensive
|
||||
return data
|
||||
try:
|
||||
img = Image.open(BytesIO(data))
|
||||
orientation = img.getexif().get(_EXIF_ORIENTATION_TAG)
|
||||
if not orientation or orientation == 1:
|
||||
return data # upright already — keep the original bytes verbatim
|
||||
if img.size[0] * img.size[1] > _MAX_IMAGE_PIXELS:
|
||||
log.warning("orientation normalize skipped: image exceeds pixel cap")
|
||||
return data
|
||||
fmt = img.format or "PNG"
|
||||
upright = ImageOps.exif_transpose(img) # applies the rotation + drops the tag
|
||||
if upright is None: # pragma: no cover - in_place=False never returns None
|
||||
return data
|
||||
buf = BytesIO()
|
||||
save_kwargs: dict[str, object] = {}
|
||||
if fmt in ("JPEG", "WEBP"):
|
||||
save_kwargs["quality"] = 90
|
||||
upright.save(buf, format=fmt, **save_kwargs)
|
||||
return buf.getvalue()
|
||||
except Exception as exc:
|
||||
log.warning("orientation normalize failed: %s", exc)
|
||||
return data
|
||||
@@ -302,7 +302,12 @@ def attachment_to_content_part(att: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raw = att.get("content")
|
||||
mime = att.get("mime_type") or "application/octet-stream"
|
||||
if kind == "image" and isinstance(raw, bytes):
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
from turnstone.core.images import normalize_image_orientation
|
||||
|
||||
# Bake EXIF orientation into the pixels — the model's image decoder, like
|
||||
# Pillow, ignores the orientation tag, so a phone photo would otherwise be
|
||||
# perceived sideways. Unrotated images pass through untouched.
|
||||
b64 = base64.b64encode(normalize_image_orientation(raw)).decode("ascii")
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:{mime};base64,{b64}"},
|
||||
|
||||
@@ -24,7 +24,7 @@ _MAX_IMAGE_PIXELS = 40_000_000
|
||||
def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> bytes | None:
|
||||
"""Return a small PNG thumbnail for an ``image``/``pdf`` blob, else ``None``."""
|
||||
try:
|
||||
from PIL import Image
|
||||
from PIL import Image, ImageOps
|
||||
except ImportError: # pragma: no cover - declared dependency; defensive
|
||||
log.warning("Pillow not installed; thumbnails unavailable")
|
||||
return None
|
||||
@@ -44,11 +44,11 @@ def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> by
|
||||
img = Image.open(BytesIO(data))
|
||||
else:
|
||||
return None
|
||||
# Reject oversized images explicitly before decoding. Pillow's
|
||||
# Reject oversized images explicitly before any decode. Pillow's
|
||||
# MAX_IMAGE_PIXELS only *raises* above 2x the cap; between the cap and 2x
|
||||
# it merely warns and decodes fully (a 40-80M px image → ~480MB RGB),
|
||||
# defeating the bound. The header-declared size is known after open(),
|
||||
# so gate on it before convert() — nothing past the cap is ever decoded.
|
||||
# so gate on it before exif_transpose / convert (both decode the pixels).
|
||||
# (Explicit check, not a warnings filter: make_thumbnail runs in a thread
|
||||
# and the global warnings state is not thread-safe.)
|
||||
px = img.size[0] * img.size[1]
|
||||
@@ -61,7 +61,11 @@ def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> by
|
||||
_MAX_IMAGE_PIXELS,
|
||||
)
|
||||
return None
|
||||
rgb = img.convert("RGB")
|
||||
# Honour EXIF orientation so a phone photo's thumbnail isn't rotated:
|
||||
# Pillow doesn't auto-apply the tag and PNG can't carry it. No-op for
|
||||
# the rasterized-PDF branch (its pages carry no EXIF).
|
||||
oriented = ImageOps.exif_transpose(img) or img
|
||||
rgb = oriented.convert("RGB")
|
||||
rgb.thumbnail((max_px, max_px))
|
||||
buf = BytesIO()
|
||||
rgb.save(buf, format="PNG")
|
||||
|
||||
Reference in New Issue
Block a user