fix(attachments): normalize EXIF orientation so thumbnails and models see upright images

Phone photos store landscape pixels plus an EXIF orientation tag. Browsers honour
the tag for <img>, but Pillow (our thumbnails) and many vision-model image
decoders do not — so the thumbnail rendered rotated AND the model literally
perceived the photo sideways (noticed earlier as model "hallucinations", before
thumbnails made the rotation visible).

Normalize on read, at both surfaces:
- new core/images.normalize_image_orientation: bakes the rotation into the pixels
  and re-encodes (preserving format); images with no / identity orientation pass
  through untouched (pristine original, no per-send cost).
- make_thumbnail applies exif_transpose — after the decompression-bomb pixel gate,
  which now also covers the transpose decode.
- attachment_to_content_part runs image bytes through the normalizer before
  base64, so the primary model and the perception model both get upright pixels.

Because normalization is on read (not at upload), it fixes already-stored uploads
too.
This commit is contained in:
Patrick Buckley
2026-06-16 02:05:18 -07:00
parent 793c5518cc
commit 0171a9dd18
5 changed files with 140 additions and 5 deletions
+49
View File
@@ -0,0 +1,49 @@
"""Tests for EXIF-orientation normalisation (turnstone.core.images)."""
from __future__ import annotations
from io import BytesIO
import pytest
from turnstone.core.images import normalize_image_orientation
Image = pytest.importorskip("PIL.Image")
_ORIENTATION_TAG = 0x0112 # EXIF orientation (standard tag id)
def _oriented_jpeg(orientation: int, size: tuple[int, int] = (4, 2)) -> bytes:
img = Image.new("RGB", size, "red")
exif = img.getexif()
exif[_ORIENTATION_TAG] = orientation
buf = BytesIO()
img.save(buf, format="JPEG", exif=exif)
return buf.getvalue()
def test_applies_rotation_and_strips_tag() -> None:
# Orientation 6 = "rotate 90° for display": a 4×2 landscape becomes 2×4.
data = _oriented_jpeg(6, size=(4, 2))
out = normalize_image_orientation(data)
assert out != data, "a rotated image must be re-encoded upright"
img = Image.open(BytesIO(out))
assert img.size == (2, 4), "the 90° rotation must be baked into the pixels"
assert img.getexif().get(_ORIENTATION_TAG) in (None, 1), "the orientation tag must be cleared"
def test_passthrough_when_upright() -> None:
data = _oriented_jpeg(1, size=(4, 2))
assert normalize_image_orientation(data) == data, "identity orientation must not re-encode"
def test_passthrough_when_no_exif() -> None:
buf = BytesIO()
Image.new("RGB", (3, 3), "blue").save(buf, format="PNG")
data = buf.getvalue()
assert normalize_image_orientation(data) == data, "a tag-less image must pass through verbatim"
def test_never_raises_on_garbage() -> None:
assert normalize_image_orientation(b"not an image") == b"not an image"
assert normalize_image_orientation(b"") == b""
+14
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from io import BytesIO
import pytest
from turnstone.core.thumbnails import make_thumbnail
@@ -42,6 +44,18 @@ class TestMakeThumbnail:
out = make_thumbnail(PNG_1x1, "image")
assert out is not None and out[:8] == _PNG_MAGIC
def test_image_thumbnail_honours_exif_orientation(self) -> None:
pil = pytest.importorskip("PIL.Image")
src = pil.new("RGB", (40, 20), "red") # landscape source
exif = src.getexif()
exif[0x0112] = 6 # "rotate 90° for display" → the thumbnail should be portrait
buf = BytesIO()
src.save(buf, format="JPEG", exif=exif)
out = make_thumbnail(buf.getvalue(), "image")
assert out is not None
thumb = pil.open(BytesIO(out))
assert thumb.height > thumb.width, "thumbnail must reflect the applied EXIF rotation"
def test_pdf_thumbnail_is_png(self) -> None:
out = make_thumbnail(_minimal_pdf(), "pdf")
assert out is not None and out[:8] == _PNG_MAGIC
+63
View File
@@ -0,0 +1,63 @@
"""Image-pixel utilities shared across the thumbnail, wire, and perception paths.
Currently: EXIF-orientation normalisation. Kept separate from
:mod:`turnstone.core.thumbnails` (which downscales for the UI) — this operates on
full-resolution bytes at the read/wire boundary so every consumer of an
attachment sees the same upright pixels.
"""
from __future__ import annotations
from io import BytesIO
from turnstone.core.log import get_logger
log = get_logger(__name__)
# EXIF tag 0x0112 (274) — image orientation (1 = upright; 2-8 = flips/rotations).
_EXIF_ORIENTATION_TAG = 0x0112
# Mirror turnstone.core.thumbnails: bound decoded pixels so a small compressed
# file that expands to an enormous bitmap can't OOM the node during re-encode.
_MAX_IMAGE_PIXELS = 40_000_000
def normalize_image_orientation(data: bytes) -> bytes:
"""Bake an image's EXIF orientation into its pixels; return re-encoded bytes.
Images with no orientation tag (or an identity orientation) are returned
UNCHANGED — no decode/re-encode, so the pristine original is preserved and
there is no per-send cost in the common case. Never raises: any failure
(Pillow missing, decode error, oversized) returns the original bytes.
Why this exists: a phone photo stores landscape pixels plus an orientation
tag. Browsers honour the tag for ``<img>``, but Pillow (our thumbnails) and
many vision-model image decoders do NOT — so the model literally perceives
the photo rotated. Normalising at the read/wire boundary makes every
consumer (browser, thumbnail, model) see the same upright image.
"""
try:
from PIL import Image, ImageOps
except ImportError: # pragma: no cover - declared dependency; defensive
return data
try:
img = Image.open(BytesIO(data))
orientation = img.getexif().get(_EXIF_ORIENTATION_TAG)
if not orientation or orientation == 1:
return data # upright already — keep the original bytes verbatim
if img.size[0] * img.size[1] > _MAX_IMAGE_PIXELS:
log.warning("orientation normalize skipped: image exceeds pixel cap")
return data
fmt = img.format or "PNG"
upright = ImageOps.exif_transpose(img) # applies the rotation + drops the tag
if upright is None: # pragma: no cover - in_place=False never returns None
return data
buf = BytesIO()
save_kwargs: dict[str, object] = {}
if fmt in ("JPEG", "WEBP"):
save_kwargs["quality"] = 90
upright.save(buf, format=fmt, **save_kwargs)
return buf.getvalue()
except Exception as exc:
log.warning("orientation normalize failed: %s", exc)
return data
+6 -1
View File
@@ -302,7 +302,12 @@ def attachment_to_content_part(att: dict[str, Any]) -> dict[str, Any] | None:
raw = att.get("content")
mime = att.get("mime_type") or "application/octet-stream"
if kind == "image" and isinstance(raw, bytes):
b64 = base64.b64encode(raw).decode("ascii")
from turnstone.core.images import normalize_image_orientation
# Bake EXIF orientation into the pixels — the model's image decoder, like
# Pillow, ignores the orientation tag, so a phone photo would otherwise be
# perceived sideways. Unrotated images pass through untouched.
b64 = base64.b64encode(normalize_image_orientation(raw)).decode("ascii")
return {
"type": "image_url",
"image_url": {"url": f"data:{mime};base64,{b64}"},
+8 -4
View File
@@ -24,7 +24,7 @@ _MAX_IMAGE_PIXELS = 40_000_000
def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> bytes | None:
"""Return a small PNG thumbnail for an ``image``/``pdf`` blob, else ``None``."""
try:
from PIL import Image
from PIL import Image, ImageOps
except ImportError: # pragma: no cover - declared dependency; defensive
log.warning("Pillow not installed; thumbnails unavailable")
return None
@@ -44,11 +44,11 @@ def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> by
img = Image.open(BytesIO(data))
else:
return None
# Reject oversized images explicitly before decoding. Pillow's
# Reject oversized images explicitly before any decode. Pillow's
# MAX_IMAGE_PIXELS only *raises* above 2x the cap; between the cap and 2x
# it merely warns and decodes fully (a 40-80M px image → ~480MB RGB),
# defeating the bound. The header-declared size is known after open(),
# so gate on it before convert() — nothing past the cap is ever decoded.
# so gate on it before exif_transpose / convert (both decode the pixels).
# (Explicit check, not a warnings filter: make_thumbnail runs in a thread
# and the global warnings state is not thread-safe.)
px = img.size[0] * img.size[1]
@@ -61,7 +61,11 @@ def make_thumbnail(data: bytes, kind: str, *, max_px: int = _THUMB_MAX_PX) -> by
_MAX_IMAGE_PIXELS,
)
return None
rgb = img.convert("RGB")
# Honour EXIF orientation so a phone photo's thumbnail isn't rotated:
# Pillow doesn't auto-apply the tag and PNG can't carry it. No-op for
# the rasterized-PDF branch (its pages carry no EXIF).
oriented = ImageOps.exif_transpose(img) or img
rgb = oriented.convert("RGB")
rgb.thumbnail((max_px, max_px))
buf = BytesIO()
rgb.save(buf, format="PNG")