diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 146eabce23..fec2b01802 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -33,7 +33,7 @@ enum resize_algo { RESIZE_ALGO_BILINEAR, // stretch to target resolution RESIZE_ALGO_BICUBIC, // center-crop when aspect ratio doesn't match RESIZE_ALGO_BICUBIC_PILLOW, - // RESIZE_ALGO_LANCZOS, // TODO + RESIZE_ALGO_LANCZOS, }; // Padding style for img_tool::resize diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 36cd463b20..72d35fce69 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -68,6 +68,9 @@ struct img_tool { case RESIZE_ALGO_BICUBIC_PILLOW: resize_bicubic_pillow(src, dst, target_resolution.width, target_resolution.height); break; + case RESIZE_ALGO_LANCZOS: + resize_lanczos_pillow(src, dst, target_resolution.width, target_resolution.height); + break; default: throw std::runtime_error("Unsupported resize algorithm"); } @@ -97,6 +100,9 @@ struct img_tool { case RESIZE_ALGO_BICUBIC_PILLOW: resize_bicubic_pillow(src, resized_image, new_width, new_height); break; + case RESIZE_ALGO_LANCZOS: + resize_lanczos_pillow(src, resized_image, new_width, new_height); + break; default: throw std::runtime_error("Unsupported resize algorithm"); } @@ -337,22 +343,50 @@ private: } } - // Bicubic resize function using Pillow's ImagingResample algorithm + // Pillow-compatible separable resampling (Bicubic and Lanczos) // Adapted from https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Resample.c // - // Key Difference with resize_bicubic: - // 1. Uses separable filtering: horizontal pass followed by vertical pass + // Key properties: + // 1. Separable filtering: horizontal pass followed by vertical pass // 2. Pre-computes normalized filter coefficients for each output pixel - // 3. Applies convolution using fixed-point integer arithmetic for performance + // 3. Fixed-point integer arithmetic (22 fractional bits) for speed and determinism static bool resize_bicubic_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) { + return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/false); + } + + // Lanczos-3 (support radius 3), matches Pillow's Image.LANCZOS + static bool resize_lanczos_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) { + return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/true); + } + + static bool resize_pillow( + const clip_image_u8 & img, + clip_image_u8 & dst, + int target_width, + int target_height, + bool use_lanczos) { // Fixed-point precision: 22 bits = 32 (int32_t) - 8 (uint8_t pixels) - 2 (headroom for accumulation) // This allows encoding fractional weights as integers: weight * 2^22 const int PRECISION_BITS = 32 - 8 - 2; - // Bicubic filter function with a = -0.5 (Note that GGML/PyTorch takes a = -0.75) + // Resample filter: Lanczos-3 (support [-3, 3]) or bicubic with a = -0.5 (support [-2, 2]) + // Note: GGML/PyTorch bicubic uses a = -0.75, Pillow uses a = -0.5 // Returns filter weight for distance x from pixel center - // Support: [-2, 2], meaning the filter influences pixels within 2 units of distance - auto bicubic_filter = [](double x) -> double { + auto resample_filter = [use_lanczos](double x) -> double { + if (use_lanczos) { + if (-3.0 <= x && x < 3.0) { + auto sinc = [](double v) { + if (v == 0.0) { + return 1.0; + } + const double pi_v = v * 3.141592653589793238462643383279502884; + return std::sin(pi_v) / pi_v; + }; + return sinc(x) * sinc(x / 3.0); + } + return 0.0; + } + constexpr double a = -0.5; if (x < 0.0) { x = -x; @@ -366,8 +400,8 @@ private: return 0.0; // Zero outside [-2, 2] }; - // Filter support radius: bicubic extends 2 pixels in each direction - constexpr double filter_support = 2.0; + // Filter support radius: 2 for bicubic, 3 for lanczos + const double filter_support = use_lanczos ? 3.0 : 2.0; // Clipping function for 8-bit values auto clip8 = [](int val) -> uint8_t { @@ -434,7 +468,7 @@ private: // Compute filter weights for each contributing input pixel for (x = 0; x < xmax; x++) { // Distance from input pixel center to output pixel center in input space - double w = bicubic_filter((x + xmin - center + 0.5) * ss); + double w = resample_filter((x + xmin - center + 0.5) * ss); pre_weights[xx * ksize + x] = w; ww += w; // Accumulate for normalization } @@ -463,6 +497,12 @@ private: const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS for (int i = 0; i < outSize * ksize; i++) { + if (use_lanczos) { + // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice + const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5); + weights[i] = static_cast(rounded); + continue; + } double tmp_val = pre_weights[i] * fxp_scale; if (pre_weights[i] < 0) { tmp_val -= 0.5;