Skip to content

Commit

Permalink
choose gainmap application space adaptively basing on input gamuts
Browse files Browse the repository at this point in the history
during generateGainMap, the color spaces of hdr intent and sdr intent
are unified. if a wider gamut space is converted to narrower-gamut this
may result in pixel values less than zero and/or greater than 1.
choosing offsets to keep the domain of log during gainmap computation is
tricky. This can be mitigated by converting narrower gamut space data to
wider gamut space. This yields lesser overrange values. current change
implements this.

for this to work as intended writing xmp metadata in the bitstream needs
to be disabled.

Test: ./ultrahdr_app options
Change-Id: I0155a1dc043f3c3e0493e04e5a6eabe160659476
  • Loading branch information
ram-mohan committed Nov 28, 2024
1 parent 5bbe947 commit dd5452e
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 38 deletions.
7 changes: 4 additions & 3 deletions lib/include/ultrahdr/ultrahdrcommon.h
Original file line number Diff line number Diff line change
Expand Up @@ -204,20 +204,21 @@ typedef struct uhdr_effect_desc uhdr_effect_desc_t;
typedef struct uhdr_gainmap_metadata_ext : uhdr_gainmap_metadata {
uhdr_gainmap_metadata_ext() {}

uhdr_gainmap_metadata_ext(std::string ver) { version = ver; }
uhdr_gainmap_metadata_ext(std::string ver) : version(ver), use_base_cg(true) {}

uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver) {
uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver)
: uhdr_gainmap_metadata_ext(ver) {
max_content_boost = metadata.max_content_boost;
min_content_boost = metadata.min_content_boost;
gamma = metadata.gamma;
offset_sdr = metadata.offset_sdr;
offset_hdr = metadata.offset_hdr;
hdr_capacity_min = metadata.hdr_capacity_min;
hdr_capacity_max = metadata.hdr_capacity_max;
version = ver;
}

std::string version; /**< Ultra HDR format version */
bool use_base_cg; /**< Is gainmap application space base color space */
} uhdr_gainmap_metadata_ext_t; /**< alias for struct uhdr_gainmap_metadata */

#ifdef UHDR_ENABLE_GLES
Expand Down
13 changes: 2 additions & 11 deletions lib/src/gainmapmetadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -344,16 +344,6 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat(
return status;
}

// TODO: parse gainmap image icc and use it for color conversion during applygainmap
if (!from->useBaseColorSpace) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"current implementation requires gainmap application space to match base color space");
return status;
}

to->version = kJpegrVersion;
to->max_content_boost = exp2((float)from->gainMapMaxN[0] / from->gainMapMaxD[0]);
to->min_content_boost = exp2((float)from->gainMapMinN[0] / from->gainMapMinD[0]);
Expand All @@ -365,6 +355,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat(
to->offset_hdr = (float)from->alternateOffsetN[0] / from->alternateOffsetD[0];
to->hdr_capacity_max = exp2((float)from->alternateHdrHeadroomN / from->alternateHdrHeadroomD);
to->hdr_capacity_min = exp2((float)from->baseHdrHeadroomN / from->baseHdrHeadroomD);
to->use_base_cg = from->useBaseColorSpace;

return g_no_error;
}
Expand All @@ -381,7 +372,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction(
}

to->backwardDirection = false;
to->useBaseColorSpace = true;
to->useBaseColorSpace = from->use_base_cg;

#define CONVERT_FLT_TO_UNSIGNED_FRACTION(flt, numerator, denominator) \
if (!floatToUnsignedFraction(flt, numerator, denominator)) { \
Expand Down
14 changes: 11 additions & 3 deletions lib/src/gpu/applygainmap_gl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ std::string getGamutConversionShader(uhdr_color_gamut_t src_cg, uhdr_color_gamut

std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_fmt,
uhdr_color_transfer output_ct, uhdr_color_gamut_t sdr_cg,
uhdr_color_gamut_t hdr_cg) {
uhdr_color_gamut_t hdr_cg, bool use_base_cg) {
std::string shader_code = R"__SHADER__(#version 300 es
precision highp float;
precision highp int;
Expand Down Expand Up @@ -278,10 +278,17 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
vec3 yuv_gamma_sdr = getYUVPixel();
vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr);
)__SHADER__");
if (sdr_cg != hdr_cg && !use_base_cg) {
shader_code.append(R"__SHADER__(
rgb_sdr = gamutConversion(rgb_sdr);
)__SHADER__");
}
shader_code.append(R"__SHADER__(
vec3 gain = sampleMap(gainMapTexture);
vec3 rgb_hdr = applyGain(rgb_sdr, gain);
)__SHADER__");
if (sdr_cg != hdr_cg) {
if (sdr_cg != hdr_cg && use_base_cg) {
shader_code.append(R"__SHADER__(
rgb_hdr = gamutConversion(rgb_hdr);
)__SHADER__");
Expand Down Expand Up @@ -354,7 +361,8 @@ uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_

shaderProgram = opengl_ctxt->create_shader_program(
vertex_shader.c_str(),
getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg)
getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg,
gainmap_metadata->use_base_cg)
.c_str());
RET_IF_ERR()

Expand Down
79 changes: 58 additions & 21 deletions lib/src/jpegr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -584,15 +584,40 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
return status;
}

ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg);
if (hdrGamutConversionFn == nullptr) {
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for gamut conversion from %d to %d", hdr_intent->cg,
sdr_intent->cg);
return status;
ColorTransformFn hdrGamutConversionFn;
ColorTransformFn sdrGamutConversionFn;
bool use_sdr_cg = true;
if (sdr_intent->cg != hdr_intent->cg) {
use_sdr_cg = kWriteXmpMetadata ||
!(hdr_intent->cg == UHDR_CG_BT_2100 ||
(hdr_intent->cg == UHDR_CG_DISPLAY_P3 && sdr_intent->cg != UHDR_CG_BT_2100));
if (use_sdr_cg) {
hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg);
if (hdrGamutConversionFn == nullptr) {
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for gamut conversion from %d to %d", hdr_intent->cg,
sdr_intent->cg);
return status;
}
sdrGamutConversionFn = identityConversion;
} else {
hdrGamutConversionFn = identityConversion;
sdrGamutConversionFn = getGamutConversionFn(hdr_intent->cg, sdr_intent->cg);
if (sdrGamutConversionFn == nullptr) {
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for gamut conversion from %d to %d", sdr_intent->cg,
hdr_intent->cg);
return status;
}
}
} else {
hdrGamutConversionFn = sdrGamutConversionFn = identityConversion;
}
gainmap_metadata->use_base_cg = use_sdr_cg;

ColorTransformFn sdrYuvToRgbFn = getYuvToRgbFn(sdr_intent->cg);
if (sdrYuvToRgbFn == nullptr) {
Expand Down Expand Up @@ -675,8 +700,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_

auto generateGainMapOnePass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_height,
hdrInvOetf, hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn,
luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn,
hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits,
use_luminance]() -> void {
gainmap_metadata->max_content_boost = hdr_white_nits / kSdrWhiteNits;
gainmap_metadata->min_content_boost = 1.0f;
gainmap_metadata->gamma = mGamma;
Expand All @@ -698,9 +724,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
JobQueue jobQueue;
std::function<void()> generateMap =
[this, sdr_intent, hdr_intent, gainmap_metadata, dest, hdrInvOetf, hdrLuminanceFn,
hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost, log2MaxBoost,
use_luminance, &jobQueue]() -> void {
hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn,
hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost,
log2MaxBoost, use_luminance, &jobQueue]() -> void {
unsigned int rowStart, rowEnd;
const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt);
const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt);
Expand All @@ -727,6 +753,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
#else
Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
#endif
sdr_rgb = sdrGamutConversionFn(sdr_rgb);
sdr_rgb = clipNegatives(sdr_rgb);

Color hdr_rgb_gamma;

Expand Down Expand Up @@ -791,10 +819,11 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); });
};

auto generateGainMapTwoPass =
[this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width, map_height, hdrInvOetf,
hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
auto generateGainMapTwoPass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width,
map_height, hdrInvOetf, hdrLuminanceFn, hdrOotfFn,
hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn,
sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn,
hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
uhdr_memory_block_t gainmap_mem((size_t)map_width * map_height * sizeof(float) *
(mUseMultiChannelGainMap ? 3 : 1));
float* gainmap_data = reinterpret_cast<float*>(gainmap_mem.m_buffer.get());
Expand All @@ -808,9 +837,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
JobQueue jobQueue;
std::function<void()> generateMap =
[this, sdr_intent, hdr_intent, gainmap_data, map_width, hdrInvOetf, hdrLuminanceFn,
hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance, &gainmap_min,
&gainmap_max, &gainmap_minmax, &jobQueue]() -> void {
hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn,
hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance,
&gainmap_min, &gainmap_max, &gainmap_minmax, &jobQueue]() -> void {
unsigned int rowStart, rowEnd;
const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt);
const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt);
Expand Down Expand Up @@ -840,6 +869,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
#else
Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
#endif
sdr_rgb = sdrGamutConversionFn(sdr_rgb);
sdr_rgb = clipNegatives(sdr_rgb);

Color hdr_rgb_gamma;

Expand Down Expand Up @@ -1412,8 +1443,11 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
uhdr_color_gamut_t sdr_cg =
sdr_intent->cg == UHDR_CG_UNSPECIFIED ? UHDR_CG_BT_709 : sdr_intent->cg;
uhdr_color_gamut_t hdr_cg = gainmap_img->cg == UHDR_CG_UNSPECIFIED ? sdr_cg : gainmap_img->cg;
ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(hdr_cg, sdr_cg);
dest->cg = hdr_cg;
ColorTransformFn hdrGamutConversionFn =
gainmap_metadata->use_base_cg ? getGamutConversionFn(hdr_cg, sdr_cg) : identityConversion;
ColorTransformFn sdrGamutConversionFn =
gainmap_metadata->use_base_cg ? identityConversion : getGamutConversionFn(hdr_cg, sdr_cg);

#ifdef UHDR_ENABLE_GLES
if (mUhdrGLESCtxt != nullptr) {
Expand Down Expand Up @@ -1487,6 +1521,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
JobQueue jobQueue;
std::function<void()> applyRecMap = [sdr_intent, gainmap_img, dest, &jobQueue, &idwTable,
output_ct, &gainLUT, gainmap_metadata, hdrGamutConversionFn,
sdrGamutConversionFn,
#if !USE_APPLY_GAIN_LUT
gainmap_weight,
#endif
Expand All @@ -1506,6 +1541,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
#else
Color rgb_sdr = srgbInvOetf(rgb_gamma_sdr);
#endif
rgb_sdr = sdrGamutConversionFn(rgb_sdr);
Color rgb_hdr;
if (gainmap_img->fmt == UHDR_IMG_FMT_8bppYCbCr400) {
float gain;
Expand Down Expand Up @@ -2490,6 +2526,7 @@ status_t JpegR::encodeJPEGR(jr_compressed_ptr yuv420jpg_image_ptr,

uhdr_gainmap_metadata_ext_t meta;
meta.version = metadata->version;
meta.use_base_cg = true;
meta.hdr_capacity_max = metadata->hdrCapacityMax;
meta.hdr_capacity_min = metadata->hdrCapacityMin;
meta.gamma = metadata->gamma;
Expand Down
1 change: 1 addition & 0 deletions lib/src/jpegrutils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,7 @@ uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size,
snprintf(status.detail, sizeof status.detail, "hdr intent as base rendition is not supported");
return status;
}
metadata->use_base_cg = true;

return g_no_error;
}
Expand Down
4 changes: 4 additions & 0 deletions tests/gainmapmetadata_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
expected.offset_hdr = 0.0625f;
expected.hdr_capacity_min = 1.0f;
expected.hdr_capacity_max = 10000.0f / 203.0f;
expected.use_base_cg = false;

uhdr_gainmap_metadata_frac metadata;
EXPECT_EQ(
Expand Down Expand Up @@ -78,12 +79,14 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr);
EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min);
EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max);
EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg);

data.clear();
expected.min_content_boost = 0.000578369f;
expected.offset_sdr = -0.0625f;
expected.offset_hdr = -0.0625f;
expected.hdr_capacity_max = 1000.0f / 203.0f;
expected.use_base_cg = true;

EXPECT_EQ(
uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction(&expected, &metadata).error_code,
Expand All @@ -104,6 +107,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr);
EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min);
EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max);
EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg);
}

} // namespace ultrahdr
2 changes: 2 additions & 0 deletions tests/jpegr_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1403,6 +1403,7 @@ TEST(JpegRTest, DecodeAPIWithInvalidArgs) {
TEST(JpegRTest, writeXmpThenRead) {
uhdr_gainmap_metadata_ext_t metadata_expected;
metadata_expected.version = "1.0";
metadata_expected.use_base_cg = true;
metadata_expected.max_content_boost = 1.25f;
metadata_expected.min_content_boost = 0.75f;
metadata_expected.gamma = 1.0f;
Expand Down Expand Up @@ -1432,6 +1433,7 @@ TEST(JpegRTest, writeXmpThenRead) {
EXPECT_FLOAT_EQ(metadata_expected.offset_hdr, metadata_read.offset_hdr);
EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_min, metadata_read.hdr_capacity_min);
EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_max, metadata_read.hdr_capacity_max);
EXPECT_TRUE(metadata_read.use_base_cg);
}

class JpegRAPIEncodeAndDecodeTest
Expand Down

0 comments on commit dd5452e

Please sign in to comment.