feat: Remove image distortion when resizing

imgly · Mar 26, 2024 · d2a35bc · d2a35bc
1 parent 251a3da
commit d2a35bc
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 31 deletions.
diff --git a/packages/web-examples/vite-project/src/App.vue b/packages/web-examples/vite-project/src/App.vue
@@ -1,5 +1,8 @@
+<!--Try this http://localhost:5173/?auto=1&image=https://images.unsplash.com/photo-1709248835088-03bb0946d6ab -->
+http://localhost:5173/?auto=1&image=http://localhost:5173/images/tile_0.webp
 <script>
 import { ref, watch, onMounted, onUnmounted } from 'vue';
+
 import {
   preload,
   removeBackground,
@@ -12,11 +15,20 @@ export default {
   name: 'App',
   setup() {
     const images = [
-      'https://images.unsplash.com/photo-1686002359940-6a51b0d64f68?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=1024&q=80',
+      // 'https://images.unsplash.com/photo-1656408308602-05835d990fb1?q=80&w=3200&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D',
+      // 'https://images.unsplash.com/photo-1686002359940-6a51b0d64f68?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=1024&q=80',
       'https://images.unsplash.com/photo-1590523278191-995cbcda646b?ixlib=rb-1.2.1&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9',
       'https://images.unsplash.com/photo-1709248835088-03bb0946d6ab?q=80&w=3387&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D'
     ];
-    const randomImage = images[Math.floor(Math.random() * images.length)];
+
+    const url = new URL(window.location.href);
+    const params = new URLSearchParams(url.search);
+    const image = params.get('image');
+    const auto = params.get('auto') || false;
+    const randomImage = image
+      ? image
+      : images[Math.floor(Math.random() * images.length)];
+
     const imageUrl = ref(randomImage);
     const isRunning = ref(false);
     const seconds = ref(0);
@@ -123,6 +135,8 @@ export default {
       stopTimer();
     };
 
+    if (auto) load();
+
     return { imageUrl, isRunning, seconds, caption, load };
   }
 };

diff --git a/packages/web/src/inference.ts b/packages/web/src/inference.ts
@@ -28,7 +28,13 @@ async function runInference(
 ): Promise<NdArray<Uint8Array>> {
   const resolution = 1024;
   const [srcHeight, srcWidth, srcChannels] = imageTensor.shape;
-  let tensorImage = tensorResizeBilinear(imageTensor, resolution, resolution);
+  const proportional = true;
+  let tensorImage = tensorResizeBilinear(
+    imageTensor,
+    resolution,
+    resolution,
+    proportional
+  );
   const inputTensor = tensorHWCtoBCHW(tensorImage); // this converts also from float to rgba
 
   const predictionsDict = await runOnnxSession(
@@ -39,8 +45,20 @@ async function runInference(
   );
 
   let alphamask = ndarray(predictionsDict[0].data, [resolution, resolution, 1]);
+  // alphamask = tensorResizeBilinear(
+  //   alphamask,
+  //   srcWidth,
+  //   srcHeight,
+  //   proportional
+  // );
+
   let alphamaskU8 = convertFloat32ToUint8(alphamask);
-  alphamaskU8 = tensorResizeBilinear(alphamaskU8, srcWidth, srcHeight);
+  alphamaskU8 = tensorResizeBilinear(
+    alphamaskU8,
+    srcWidth,
+    srcHeight,
+    proportional
+  );
 
   return alphamaskU8;
 }
diff --git a/packages/web/src/utils.ts b/packages/web/src/utils.ts
@@ -4,13 +4,12 @@ export {
   tensorResizeBilinear,
   tensorHWCtoBCHW,
   imageBitmapToImageData,
-  calculateProportionalSize,
   imageSourceToImageData,
-  ImageSource,
+  type ImageSource,
   createCanvas
 };
 
-import ndarray, { NdArray } from 'ndarray';
+import ndarray, { NdArray, TypedArray } from 'ndarray';
 import { imageDecode, imageEncode } from './codecs';
 import { ensureAbsoluteURI } from './url';
 import { Config } from './schema';
@@ -31,19 +30,44 @@ function imageBitmapToImageData(imageBitmap: ImageBitmap): ImageData {
   return ctx.getImageData(0, 0, canvas.width, canvas.height);
 }
 
-function tensorResizeBilinear(
-  imageTensor: NdArray<Uint8Array>,
+function createTypeArray<T extends TypedArray>(length: number) {
+  if (typeof Uint8Array !== 'undefined') {
+    return new Uint8Array(length) as T;
+  } else if (typeof Uint8ClampedArray !== 'undefined') {
+    return new Uint8ClampedArray(length) as T;
+  } else if (typeof Uint16Array !== 'undefined') {
+    return new Uint16Array(length) as T;
+  } else if (typeof Uint32Array !== 'undefined') {
+    return new Uint32Array(length) as T;
+  } else if (typeof Float32Array !== 'undefined') {
+    return new Float32Array(length) as T;
+  } else if (typeof Float64Array !== 'undefined') {
+    return new Float64Array(length) as T;
+  } else {
+    throw new Error('TypedArray not supported');
+  }
+}
+function tensorResizeBilinear<T extends TypedArray>(
+  imageTensor: NdArray<T>,
   newWidth: number,
-  newHeight: number
-): NdArray<Uint8Array> {
+  newHeight: number,
+  proportional: boolean = false
+): NdArray<T> {
   const [srcHeight, srcWidth, srcChannels] = imageTensor.shape;
-  // Calculate the scaling factors
-  const scaleX = srcWidth / newWidth;
-  const scaleY = srcHeight / newHeight;
+
+  let scaleX = srcWidth / newWidth;
+  let scaleY = srcHeight / newHeight;
+
+  if (proportional) {
+    const downscaling = Math.max(scaleX, scaleY) > 1.0;
+    scaleX = scaleY = downscaling
+      ? Math.max(scaleX, scaleY)
+      : Math.min(scaleX, scaleY);
+  }
 
   // Create a new NdArray to store the resized image
   const resizedImageData = ndarray(
-    new Uint8Array(srcChannels * newWidth * newHeight),
+    createTypeArray<T>(srcChannels * newWidth * newHeight),
     [newHeight, newWidth, srcChannels]
   );
   // Perform interpolation to fill the resized NdArray
@@ -71,8 +95,8 @@ function tensorResizeBilinear(
           dx * (1 - dy) * p2 +
           (1 - dx) * dy * p3 +
           dx * dy * p4;
-
-        resizedImageData.set(y, x, c, Math.round(interpolatedValue));
+        // console.log(interpolatedValue);
+        resizedImageData.set(y, x, c, interpolatedValue);
       }
     }
   }
@@ -101,20 +125,6 @@ function tensorHWCtoBCHW(
   return ndarray(float32Data, [1, 3, srcHeight, srcWidth]);
 }
 
-function calculateProportionalSize(
-  originalWidth: number,
-  originalHeight: number,
-  maxWidth: number,
-  maxHeight: number
-): [number, number] {
-  const widthRatio = maxWidth / originalWidth;
-  const heightRatio = maxHeight / originalHeight;
-  const scalingFactor = Math.min(widthRatio, heightRatio);
-  const newWidth = Math.floor(originalWidth * scalingFactor);
-  const newHeight = Math.floor(originalHeight * scalingFactor);
-  return [newWidth, newHeight];
-}
-
 async function imageSourceToImageData(
   image: ImageSource,
   config: Config