choose gainmap application space adaptively basing on input gamuts

during generateGainMap, the color spaces of hdr intent and sdr intent
are unified. if a wider gamut space is converted to narrower-gamut this
may result in pixel values less than zero and/or greater than 1.
choosing offsets to keep the domain of log during gainmap computation is
tricky. This can be mitigated by converting narrower gamut space data to
wider gamut space. This yields lesser overrange values. current change
implements this.

for this to work as intended writing xmp metadata in the bitstream needs
to be disabled.

Test: ./ultrahdr_app options
Change-Id: I0155a1dc043f3c3e0493e04e5a6eabe160659476

lib/include/ultrahdr/gainmapmath.h

@@ -544,6 +544,12 @@ PutPixelFn putPixelFn(uhdr_img_fmt_t format);
 ////////////////////////////////////////////////////////////////////////////////
 // common utils
 
+static inline float clipNegatives(float value) { return (value < 0.0f) ? 0.0f : value; }
+
+static inline Color clipNegatives(Color e) {
+  return {{{clipNegatives(e.r), clipNegatives(e.g), clipNegatives(e.b)}}};
+}
+
 // maximum limit of normalized pixel value in float representation
 static const float kMaxPixelFloat = 1.0f;
 

lib/include/ultrahdr/ultrahdrcommon.h

@@ -204,20 +204,21 @@ typedef struct uhdr_effect_desc uhdr_effect_desc_t;
 typedef struct uhdr_gainmap_metadata_ext : uhdr_gainmap_metadata {
   uhdr_gainmap_metadata_ext() {}
 
-  uhdr_gainmap_metadata_ext(std::string ver) { version = ver; }
+  uhdr_gainmap_metadata_ext(std::string ver) : version(ver), use_base_cg(true) {}
 
-  uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver) {
+  uhdr_gainmap_metadata_ext(uhdr_gainmap_metadata& metadata, std::string ver)
+      : uhdr_gainmap_metadata_ext(ver) {
     max_content_boost = metadata.max_content_boost;
     min_content_boost = metadata.min_content_boost;
     gamma = metadata.gamma;
     offset_sdr = metadata.offset_sdr;
     offset_hdr = metadata.offset_hdr;
     hdr_capacity_min = metadata.hdr_capacity_min;
     hdr_capacity_max = metadata.hdr_capacity_max;
-    version = ver;
   }
 
   std::string version;         /**< Ultra HDR format version */
+  bool use_base_cg;            /**< Is gainmap application space base color space */
 } uhdr_gainmap_metadata_ext_t; /**< alias for struct uhdr_gainmap_metadata */
 
 #ifdef UHDR_ENABLE_GLES

lib/src/gainmapmetadata.cpp

@@ -344,16 +344,6 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat(
     return status;
   }
 
-  // TODO: parse gainmap image icc and use it for color conversion during applygainmap
-  if (!from->useBaseColorSpace) {
-    uhdr_error_info_t status;
-    status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
-    status.has_detail = 1;
-    snprintf(status.detail, sizeof status.detail,
-             "current implementation requires gainmap application space to match base color space");
-    return status;
-  }
-
   to->version = kJpegrVersion;
   to->max_content_boost = exp2((float)from->gainMapMaxN[0] / from->gainMapMaxD[0]);
   to->min_content_boost = exp2((float)from->gainMapMinN[0] / from->gainMapMinD[0]);
@@ -365,6 +355,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFractionToFloat(
   to->offset_hdr = (float)from->alternateOffsetN[0] / from->alternateOffsetD[0];
   to->hdr_capacity_max = exp2((float)from->alternateHdrHeadroomN / from->alternateHdrHeadroomD);
   to->hdr_capacity_min = exp2((float)from->baseHdrHeadroomN / from->baseHdrHeadroomD);
+  to->use_base_cg = from->useBaseColorSpace;
 
   return g_no_error;
 }
@@ -381,7 +372,7 @@ uhdr_error_info_t uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction(
   }
 
   to->backwardDirection = false;
-  to->useBaseColorSpace = true;
+  to->useBaseColorSpace = from->use_base_cg;
 
 #define CONVERT_FLT_TO_UNSIGNED_FRACTION(flt, numerator, denominator)                          \
   if (!floatToUnsignedFraction(flt, numerator, denominator)) {                                 \

lib/src/gpu/applygainmap_gl.cpp

@@ -244,7 +244,7 @@ std::string getGamutConversionShader(uhdr_color_gamut_t src_cg, uhdr_color_gamut
 
 std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_fmt,
                                           uhdr_color_transfer output_ct, uhdr_color_gamut_t sdr_cg,
-                                          uhdr_color_gamut_t hdr_cg) {
+                                          uhdr_color_gamut_t hdr_cg, bool use_base_cg) {
   std::string shader_code = R"__SHADER__(#version 300 es
     precision highp float;
     precision highp int;
@@ -278,10 +278,17 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
       vec3 yuv_gamma_sdr = getYUVPixel();
       vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
       vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr);
+  )__SHADER__");
+  if (sdr_cg != hdr_cg && !use_base_cg) {
+    shader_code.append(R"__SHADER__(
+      rgb_sdr = gamutConversion(rgb_sdr);
+    )__SHADER__");
+  }
+  shader_code.append(R"__SHADER__(
       vec3 gain = sampleMap(gainMapTexture);
       vec3 rgb_hdr = applyGain(rgb_sdr, gain);
   )__SHADER__");
-  if (sdr_cg != hdr_cg) {
+  if (sdr_cg != hdr_cg && use_base_cg) {
     shader_code.append(R"__SHADER__(
       rgb_hdr = gamutConversion(rgb_hdr);
     )__SHADER__");
@@ -354,7 +361,8 @@ uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_
 
   shaderProgram = opengl_ctxt->create_shader_program(
       vertex_shader.c_str(),
-      getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg)
+      getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct, sdr_cg, hdr_cg,
+                                    gainmap_metadata->use_base_cg)
           .c_str());
   RET_IF_ERR()
 

lib/src/jpegr.cpp

@@ -584,15 +584,40 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
     return status;
   }
 
-  ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg);
-  if (hdrGamutConversionFn == nullptr) {
-    status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
-    status.has_detail = 1;
-    snprintf(status.detail, sizeof status.detail,
-             "No implementation available for gamut conversion from %d to %d", hdr_intent->cg,
-             sdr_intent->cg);
-    return status;
+  ColorTransformFn hdrGamutConversionFn;
+  ColorTransformFn sdrGamutConversionFn;
+  bool use_sdr_cg = true;
+  if (sdr_intent->cg != hdr_intent->cg) {
+    use_sdr_cg = kWriteXmpMetadata ||
+                 !(hdr_intent->cg == UHDR_CG_BT_2100 ||
+                   (hdr_intent->cg == UHDR_CG_DISPLAY_P3 && sdr_intent->cg != UHDR_CG_BT_2100));
+    if (use_sdr_cg) {
+      hdrGamutConversionFn = getGamutConversionFn(sdr_intent->cg, hdr_intent->cg);
+      if (hdrGamutConversionFn == nullptr) {
+        status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
+        status.has_detail = 1;
+        snprintf(status.detail, sizeof status.detail,
+                 "No implementation available for gamut conversion from %d to %d", hdr_intent->cg,
+                 sdr_intent->cg);
+        return status;
+      }
+      sdrGamutConversionFn = identityConversion;
+    } else {
+      hdrGamutConversionFn = identityConversion;
+      sdrGamutConversionFn = getGamutConversionFn(hdr_intent->cg, sdr_intent->cg);
+      if (sdrGamutConversionFn == nullptr) {
+        status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
+        status.has_detail = 1;
+        snprintf(status.detail, sizeof status.detail,
+                 "No implementation available for gamut conversion from %d to %d", sdr_intent->cg,
+                 hdr_intent->cg);
+        return status;
+      }
+    }
+  } else {
+    hdrGamutConversionFn = sdrGamutConversionFn = identityConversion;
   }
+  gainmap_metadata->use_base_cg = use_sdr_cg;
 
   ColorTransformFn sdrYuvToRgbFn = getYuvToRgbFn(sdr_intent->cg);
   if (sdrYuvToRgbFn == nullptr) {
@@ -675,8 +700,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
 
   auto generateGainMapOnePass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_height,
                                  hdrInvOetf, hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn,
-                                 luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn,
-                                 hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
+                                 sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
+                                 sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits,
+                                 use_luminance]() -> void {
     gainmap_metadata->max_content_boost = hdr_white_nits / kSdrWhiteNits;
     gainmap_metadata->min_content_boost = 1.0f;
     gainmap_metadata->gamma = mGamma;
@@ -698,9 +724,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
     JobQueue jobQueue;
     std::function<void()> generateMap =
         [this, sdr_intent, hdr_intent, gainmap_metadata, dest, hdrInvOetf, hdrLuminanceFn,
-         hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
-         sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost, log2MaxBoost,
-         use_luminance, &jobQueue]() -> void {
+         hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn,
+         hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, log2MinBoost,
+         log2MaxBoost, use_luminance, &jobQueue]() -> void {
       unsigned int rowStart, rowEnd;
       const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt);
       const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt);
@@ -727,6 +753,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
 #else
             Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
 #endif
+            sdr_rgb = sdrGamutConversionFn(sdr_rgb);
+            sdr_rgb = clipNegatives(sdr_rgb);
 
             Color hdr_rgb_gamma;
 
@@ -791,10 +819,11 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
     std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); });
   };
 
-  auto generateGainMapTwoPass =
-      [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width, map_height, hdrInvOetf,
-       hdrLuminanceFn, hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
-       sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
+  auto generateGainMapTwoPass = [this, sdr_intent, hdr_intent, gainmap_metadata, dest, map_width,
+                                 map_height, hdrInvOetf, hdrLuminanceFn, hdrOotfFn,
+                                 hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn,
+                                 sdrYuvToRgbFn, hdrYuvToRgbFn, sdr_sample_pixel_fn,
+                                 hdr_sample_pixel_fn, hdr_white_nits, use_luminance]() -> void {
     uhdr_memory_block_t gainmap_mem((size_t)map_width * map_height * sizeof(float) *
                                     (mUseMultiChannelGainMap ? 3 : 1));
     float* gainmap_data = reinterpret_cast<float*>(gainmap_mem.m_buffer.get());
@@ -808,9 +837,9 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
     JobQueue jobQueue;
     std::function<void()> generateMap =
         [this, sdr_intent, hdr_intent, gainmap_data, map_width, hdrInvOetf, hdrLuminanceFn,
-         hdrOotfFn, hdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn, hdrYuvToRgbFn,
-         sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance, &gainmap_min,
-         &gainmap_max, &gainmap_minmax, &jobQueue]() -> void {
+         hdrOotfFn, hdrGamutConversionFn, sdrGamutConversionFn, luminanceFn, sdrYuvToRgbFn,
+         hdrYuvToRgbFn, sdr_sample_pixel_fn, hdr_sample_pixel_fn, hdr_white_nits, use_luminance,
+         &gainmap_min, &gainmap_max, &gainmap_minmax, &jobQueue]() -> void {
       unsigned int rowStart, rowEnd;
       const bool isHdrIntentRgb = isPixelFormatRgb(hdr_intent->fmt);
       const bool isSdrIntentRgb = isPixelFormatRgb(sdr_intent->fmt);
@@ -840,6 +869,8 @@ uhdr_error_info_t JpegR::generateGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_
 #else
             Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
 #endif
+            sdr_rgb = sdrGamutConversionFn(sdr_rgb);
+            sdr_rgb = clipNegatives(sdr_rgb);
 
             Color hdr_rgb_gamma;
 
@@ -1412,8 +1443,11 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
   uhdr_color_gamut_t sdr_cg =
       sdr_intent->cg == UHDR_CG_UNSPECIFIED ? UHDR_CG_BT_709 : sdr_intent->cg;
   uhdr_color_gamut_t hdr_cg = gainmap_img->cg == UHDR_CG_UNSPECIFIED ? sdr_cg : gainmap_img->cg;
-  ColorTransformFn hdrGamutConversionFn = getGamutConversionFn(hdr_cg, sdr_cg);
   dest->cg = hdr_cg;
+  ColorTransformFn hdrGamutConversionFn =
+      gainmap_metadata->use_base_cg ? getGamutConversionFn(hdr_cg, sdr_cg) : identityConversion;
+  ColorTransformFn sdrGamutConversionFn =
+      gainmap_metadata->use_base_cg ? identityConversion : getGamutConversionFn(hdr_cg, sdr_cg);
 
 #ifdef UHDR_ENABLE_GLES
   if (mUhdrGLESCtxt != nullptr) {
@@ -1487,6 +1521,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
   JobQueue jobQueue;
   std::function<void()> applyRecMap = [sdr_intent, gainmap_img, dest, &jobQueue, &idwTable,
                                        output_ct, &gainLUT, gainmap_metadata, hdrGamutConversionFn,
+                                       sdrGamutConversionFn,
 #if !USE_APPLY_GAIN_LUT
                                        gainmap_weight,
 #endif
@@ -1506,6 +1541,7 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
 #else
           Color rgb_sdr = srgbInvOetf(rgb_gamma_sdr);
 #endif
+          rgb_sdr = sdrGamutConversionFn(rgb_sdr);
           Color rgb_hdr;
           if (gainmap_img->fmt == UHDR_IMG_FMT_8bppYCbCr400) {
             float gain;
@@ -2490,6 +2526,7 @@ status_t JpegR::encodeJPEGR(jr_compressed_ptr yuv420jpg_image_ptr,
 
   uhdr_gainmap_metadata_ext_t meta;
   meta.version = metadata->version;
+  meta.use_base_cg = true;
   meta.hdr_capacity_max = metadata->hdrCapacityMax;
   meta.hdr_capacity_min = metadata->hdrCapacityMin;
   meta.gamma = metadata->gamma;

lib/src/jpegrutils.cpp

@@ -623,6 +623,7 @@ uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size,
     snprintf(status.detail, sizeof status.detail, "hdr intent as base rendition is not supported");
     return status;
   }
+  metadata->use_base_cg = true;
 
   return g_no_error;
 }

tests/gainmapmetadata_test.cpp

@@ -50,6 +50,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
   expected.offset_hdr = 0.0625f;
   expected.hdr_capacity_min = 1.0f;
   expected.hdr_capacity_max = 10000.0f / 203.0f;
+  expected.use_base_cg = false;
 
   uhdr_gainmap_metadata_frac metadata;
   EXPECT_EQ(
@@ -78,12 +79,14 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
   EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr);
   EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min);
   EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max);
+  EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg);
 
   data.clear();
   expected.min_content_boost = 0.000578369f;
   expected.offset_sdr = -0.0625f;
   expected.offset_hdr = -0.0625f;
   expected.hdr_capacity_max = 1000.0f / 203.0f;
+  expected.use_base_cg = true;
 
   EXPECT_EQ(
       uhdr_gainmap_metadata_frac::gainmapMetadataFloatToFraction(&expected, &metadata).error_code,
@@ -104,6 +107,7 @@ TEST_F(GainMapMetadataTest, encodeMetadataThenDecode) {
   EXPECT_FLOAT_EQ(expected.offset_hdr, decodedUHdrMetadata.offset_hdr);
   EXPECT_FLOAT_EQ(expected.hdr_capacity_min, decodedUHdrMetadata.hdr_capacity_min);
   EXPECT_FLOAT_EQ(expected.hdr_capacity_max, decodedUHdrMetadata.hdr_capacity_max);
+  EXPECT_EQ(expected.use_base_cg, decodedUHdrMetadata.use_base_cg);
 }
 
 }  // namespace ultrahdr

tests/jpegr_test.cpp

@@ -1403,6 +1403,7 @@ TEST(JpegRTest, DecodeAPIWithInvalidArgs) {
 TEST(JpegRTest, writeXmpThenRead) {
   uhdr_gainmap_metadata_ext_t metadata_expected;
   metadata_expected.version = "1.0";
+  metadata_expected.use_base_cg = true;
   metadata_expected.max_content_boost = 1.25f;
   metadata_expected.min_content_boost = 0.75f;
   metadata_expected.gamma = 1.0f;
@@ -1432,6 +1433,7 @@ TEST(JpegRTest, writeXmpThenRead) {
   EXPECT_FLOAT_EQ(metadata_expected.offset_hdr, metadata_read.offset_hdr);
   EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_min, metadata_read.hdr_capacity_min);
   EXPECT_FLOAT_EQ(metadata_expected.hdr_capacity_max, metadata_read.hdr_capacity_max);
+  EXPECT_TRUE(metadata_read.use_base_cg);
 }
 
 class JpegRAPIEncodeAndDecodeTest