Skip to content

Commit

Permalink
use matrix coeffs from icc tag while decoding
Browse files Browse the repository at this point in the history 
icc tag does not signal matrix coefficients directly. They signal color
primaries. From these color primaries, one can derive the matrix
coefficients needed for yuv to rgb conversion using, equations 39-44 of
itu h273. In this process bradford chroma adaptation matrix also plays a
role. The current change does not do all this. Instead it does a look up
mapping of icc tag to color gamut. Successfull harvesting of matrix
coefficients from icc tags is dependent on the color gamut dictionary
maintained internally. For now, this is okish. If gamut map fails, we
will default to srgb color space as per iso 21496-1 sec c.4.4

Test: ./ultrahdr_unit_test

Change-Id: I41cb5013c5d29d5ebaa535a78e45a9b487a3c122
  • Loading branch information
@ram-mohan
ram-mohan committed Nov 13, 2024
1 parent 821595d commit 25f743e
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 23 deletions.
2 changes: 2 additions & 0 deletions lib/include/ultrahdr/gainmapmath.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,8 @@ std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr(
std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr_neon(uhdr_raw_image_t* src);
#endif

uhdr_error_info_t convert_ycbcr_input_to_rgb(uhdr_raw_image_t* src, uhdr_raw_image_t* dst);

bool floatToSignedFraction(float v, int32_t* numerator, uint32_t* denominator);
bool floatToUnsignedFraction(float v, uint32_t* numerator, uint32_t* denominator);

Expand Down
66 changes: 66 additions & 0 deletions lib/src/gainmapmath.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1585,6 +1585,72 @@ std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr(uhdr_raw_image_
return dst;
}

uhdr_error_info_t convert_ycbcr_input_to_rgb(uhdr_raw_image_t* src, uhdr_raw_image_t* dst) {
if (dst->w != src->w || dst->h != src->h) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_MEM_ERROR;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"destination image dimensions %dx%d and source image dimensions %dx%d are not "
"identical for copy_raw_image",
dst->w, dst->h, src->w, src->h);
return status;
}

if (isPixelFormatRgb(src->fmt) || !isPixelFormatRgb(dst->fmt)) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"Unexpected source or destination color format, src fmt %d, dst fmt %d", src->fmt,
dst->fmt);
return status;
}

GetPixelFn get_pixel_fn = getPixelFn(src->fmt);
if (get_pixel_fn == nullptr) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for reading pixels for color format %d", src->fmt);
return status;
}

PutPixelFn put_pixel_fn = putPixelFn(dst->fmt);
if (put_pixel_fn == nullptr) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for writing pixels for color format %d", dst->fmt);
return status;
}

Color (*yuvToRgb)(Color) = nullptr;
if (src->cg == UHDR_CG_BT_709) {
yuvToRgb = srgbYuvToRgb;
} else if (src->cg == UHDR_CG_BT_2100) {
yuvToRgb = bt2100YuvToRgb;
} else if (src->cg == UHDR_CG_DISPLAY_P3) {
yuvToRgb = p3YuvToRgb;
} else {
yuvToRgb = Bt601YuvToRgb;
}

dst->cg = src->cg;
dst->ct = src->ct;
dst->range = UHDR_CR_FULL_RANGE;
for (unsigned int i = 0; i < src->h; i++) {
for (unsigned int j = 0; j < src->w; j++) {
Color yuv = get_pixel_fn(src, j, i);
Color rgb = yuvToRgb(yuv);
put_pixel_fn(dst, j, i, rgb);
}
}
return g_no_error;
}

std::unique_ptr<uhdr_raw_image_ext_t> copy_raw_image(uhdr_raw_image_t* src) {
std::unique_ptr<uhdr_raw_image_ext_t> dst = std::make_unique<ultrahdr::uhdr_raw_image_ext_t>(
src->fmt, src->cg, src->ct, src->range, src->w, src->h, 64);
Expand Down
59 changes: 51 additions & 8 deletions lib/src/gpu/applygainmap_gl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,46 @@ static const std::string getYuv420PixelShader = R"__SHADER__(
}
)__SHADER__";

static const std::string Bt601YUVToRGBShader = R"__SHADER__(
vec3 yuvToRgb(const vec3 color) {
const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
const mat3 transform = mat3(
1.0, 1.0, 1.0,
0.0, -0.34414, 1.772,
1.402, -0.71414, 0.0);
return clamp(transform * (color - offset), 0.0, 1.0);
}
)__SHADER__";

static const std::string Bt709YUVToRGBShader = R"__SHADER__(
vec3 yuvToRgb(const vec3 color) {
const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
const mat3 transform = mat3(
1.0, 1.0, 1.0,
0.0, -0.18732, 1.8556,
1.5748, -0.46812, 0.0);
return clamp(transform * (color - offset), 0.0, 1.0);
}
)__SHADER__";

static const std::string p3YUVToRGBShader = R"__SHADER__(
vec3 p3YuvToRgb(const vec3 color) {
vec3 yuvToRgb(const vec3 color) {
const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
const mat3 transform = mat3(
1.0, 1.0, 1.0,
0.0, -0.344136286, 1.772,
1.402, -0.714136286, 0.0);
0.0, -0.21106, 1.841426,
1.542051, -0.51044, 0.0);
return clamp(transform * (color - offset), 0.0, 1.0);
}
)__SHADER__";

static const std::string Bt2100YUVToRGBShader = R"__SHADER__(
vec3 yuvToRgb(const vec3 color) {
const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
const mat3 transform = mat3(
1.0, 1.0, 1.0,
0.0, -0.16455, 1.8814,
1.4746, -0.57135, 0.0);
return clamp(transform * (color - offset), 0.0, 1.0);
}
)__SHADER__";
Expand Down Expand Up @@ -200,7 +233,8 @@ static const std::string IdentityInverseOOTFShader = R"__SHADER__(
)__SHADER__";

std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_fmt,
uhdr_color_transfer output_ct) {
uhdr_color_transfer output_ct,
uhdr_color_gamut_t sdr_cg) {
std::string shader_code = R"__SHADER__(#version 300 es
precision highp float;
precision highp int;
Expand All @@ -216,7 +250,15 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
} else if (sdr_fmt == UHDR_IMG_FMT_12bppYCbCr420) {
shader_code.append(getYuv420PixelShader);
}
shader_code.append(p3YUVToRGBShader);
if (sdr_cg == UHDR_CG_BT_709) {
shader_code.append(Bt709YUVToRGBShader);
} else if (sdr_cg == UHDR_CG_DISPLAY_P3) {
shader_code.append(p3YUVToRGBShader);
} else if (sdr_cg == UHDR_CG_BT_2100) {
shader_code.append(Bt2100YUVToRGBShader);
} else {
shader_code.append(Bt709YUVToRGBShader);
}
shader_code.append(sRGBEOTFShader);
shader_code.append(gm_fmt == UHDR_IMG_FMT_8bppYCbCr400 ? getGainMapSampleSingleChannel
: getGainMapSampleMultiChannel);
Expand All @@ -235,7 +277,7 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
shader_code.append(R"__SHADER__(
void main() {
vec3 yuv_gamma_sdr = getYUVPixel();
vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
vec3 rgb_gamma_sdr = yuvToRgb(yuv_gamma_sdr);
vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr);
vec3 gain = sampleMap(gainMapTexture);
vec3 rgb_hdr = applyGain(rgb_sdr, gain);
Expand Down Expand Up @@ -294,7 +336,8 @@ uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_

shaderProgram = opengl_ctxt->create_shader_program(
vertex_shader.c_str(),
getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct).c_str());
getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_intent->cg)
.c_str());
RET_IF_ERR()

yuvTexture = opengl_ctxt->create_texture(sdr_intent->fmt, sdr_intent->w, sdr_intent->h,
Expand Down Expand Up @@ -364,7 +407,7 @@ uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_
opengl_ctxt->check_gl_errors("reading gles output");
RET_IF_ERR()

dest->cg = sdr_intent->cg;
dest->cg = sdr_intent->cg == UHDR_CG_UNSPECIFIED ? UHDR_CG_BT_709 : sdr_intent->cg;

if (frameBuffer) glDeleteFramebuffers(1, &frameBuffer);
if (yuvTexture) glDeleteTextures(1, &yuvTexture);
Expand Down
34 changes: 19 additions & 15 deletions lib/src/jpegr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,13 +262,13 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent, uhdr_raw_imag
}

// convert to bt601 YUV encoding for JPEG encode
#if (defined(UHDR_ENABLE_INTRINSICS) && (defined(__ARM_NEON__) || defined(__ARM_NEON)))
UHDR_ERR_CHECK(
convertYuv_neon(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
#else
UHDR_ERR_CHECK(
convertYuv(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
#endif
//#if (defined(UHDR_ENABLE_INTRINSICS) && (defined(__ARM_NEON__) || defined(__ARM_NEON)))
// UHDR_ERR_CHECK(
// convertYuv_neon(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
//#else
// UHDR_ERR_CHECK(
// convertYuv(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
//#endif

// compress sdr image
JpegEncoderHelper jpeg_enc_obj_sdr;
Expand Down Expand Up @@ -1305,9 +1305,8 @@ uhdr_error_info_t JpegR::decodeJPEGR(uhdr_compressed_image_t* uhdr_compressed_im
extractPrimaryImageAndGainMap(uhdr_compressed_img, &primary_jpeg_image, &gainmap_jpeg_image))

JpegDecoderHelper jpeg_dec_obj_sdr;
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(
primary_jpeg_image.data, primary_jpeg_image.data_sz,
(output_ct == UHDR_CT_SRGB) ? DECODE_TO_RGB_CS : DECODE_TO_YCBCR_CS));
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(primary_jpeg_image.data,
primary_jpeg_image.data_sz, DECODE_TO_YCBCR_CS));

JpegDecoderHelper jpeg_dec_obj_gm;
uhdr_raw_image_t gainmap;
Expand Down Expand Up @@ -1341,7 +1340,7 @@ uhdr_error_info_t JpegR::decodeJPEGR(uhdr_compressed_image_t* uhdr_compressed_im
sdr_intent.cg =
IccHelper::readIccColorGamut(jpeg_dec_obj_sdr.getICCPtr(), jpeg_dec_obj_sdr.getICCSize());
if (output_ct == UHDR_CT_SRGB) {
UHDR_ERR_CHECK(copy_raw_image(&sdr_intent, dest));
UHDR_ERR_CHECK(convert_ycbcr_input_to_rgb(&sdr_intent, dest));
return g_no_error;
}

Expand Down Expand Up @@ -1436,7 +1435,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
float map_scale_factor = (float)sdr_intent->w / gainmap_img->w;
int map_scale_factor_rnd = (std::max)(1, (int)std::roundf(map_scale_factor));

dest->cg = sdr_intent->cg;
dest->cg = sdr_intent->cg == UHDR_CG_UNSPECIFIED ? UHDR_CG_BT_709 : sdr_intent->cg;
// Table will only be used when map scale factor is integer.
ShepardsIDW idwTable(map_scale_factor_rnd);
float display_boost = (std::min)(max_display_boost, gainmap_metadata->hdr_capacity_max);
Expand All @@ -1463,22 +1462,27 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
return status;
}

ColorTransformFn sdrYuvToRgbFn = getYuvToRgbFn(sdr_intent->cg);
// If failed to read color space, default to srgb color space as per iso 21496-1 sec C.4.4
if (sdrYuvToRgbFn == nullptr) {
sdrYuvToRgbFn = srgbYuvToRgb;
}

JobQueue jobQueue;
std::function<void()> applyRecMap = [sdr_intent, gainmap_img, dest, &jobQueue, &idwTable,
output_ct, &gainLUT, gainmap_metadata,
#if !USE_APPLY_GAIN_LUT
gainmap_weight,
#endif
map_scale_factor, get_pixel_fn]() -> void {
map_scale_factor, get_pixel_fn, sdrYuvToRgbFn]() -> void {
unsigned int width = sdr_intent->w;
unsigned int rowStart, rowEnd;

while (jobQueue.dequeueJob(rowStart, rowEnd)) {
for (size_t y = rowStart; y < rowEnd; ++y) {
for (size_t x = 0; x < width; ++x) {
Color yuv_gamma_sdr = get_pixel_fn(sdr_intent, x, y);
// Assuming the sdr image is a decoded JPEG, we should always use Rec.601 YUV coefficients
Color rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
Color rgb_gamma_sdr = sdrYuvToRgbFn(yuv_gamma_sdr);
// We are assuming the SDR base image is always sRGB transfer.
#if USE_SRGB_INVOETF_LUT
Color rgb_sdr = srgbInvOetfLUT(rgb_gamma_sdr);
Expand Down

0 comments on commit 25f743e

Please sign in to comment.