6_es_calculation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Stamina\\.hsemotion\\enet_b0_8_best_afew.pt Compose(\n",
      "    Resize(size=(224, 224), interpolation=bilinear, max_size=None, antialias=None)\n",
      "    ToTensor()\n",
      "    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from PIL import Image\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "device = 'cuda'\n",
    "\n",
    "# Define the MTCNN face detector for cropping\n",
    "# https://github.com/timesler/facenet-pytorch\n",
    "# https://www.semanticscholar.org/paper/Joint-Face-Detection-and-Alignment-Using-Multitask-Zhang-Zhang/9e60942aa15670ed9ee03af3c0ae011fa4966b7c\n",
    "\n",
    "from facenet_pytorch import MTCNN\n",
    "mtcnn = MTCNN(keep_all=False, select_largest=False, post_process=False, min_face_size=50, device=device)\n",
    "\n",
    "# Define the emotion recognition network\n",
    "# https://github.com/av-savchenko/face-emotion-recognition\n",
    "# https://www.semanticscholar.org/paper/Classifying-Emotions-and-Engagement-in-Online-Based-Savchenko-Savchenko/260d7f95ab8a562f4ff590684ef6a509b8fed316\n",
    "\n",
    "from hsemotion.facial_emotions import HSEmotionRecognizer\n",
    "model_name = 'enet_b0_8_best_afew'\n",
    "fer = HSEmotionRecognizer(model_name=model_name, device=device)\n",
    "emotion_class_n = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_emotion_scores_from_images(mtcnn_net, images, batch_size=64):\n",
    "\n",
    "    num_images = len(images)\n",
    "    all_scores = []\n",
    "\n",
    "    for start_idx in tqdm(range(0, num_images, batch_size)):\n",
    "        end_idx = min(start_idx + batch_size, num_images)\n",
    "        batch_images = images[start_idx:end_idx]\n",
    "\n",
    "        detections, _ = mtcnn_net.detect(batch_images, landmarks=False)\n",
    "\n",
    "        detections = np.clip(np.array([detection[0] if detection is not None else [-1, -1, -1, -1] for detection in detections]), 0, 223)\n",
    "\n",
    "        invalid_images = []\n",
    "\n",
    "        face_images = []\n",
    "    \n",
    "        for i, image in enumerate(batch_images):\n",
    "            bounding_box = detections[i]\n",
    "            box = bounding_box.astype(int)\n",
    "            x1, y1, x2, y2 = box[0:4]\n",
    "            if x2 == 0:\n",
    "                face_images.append(np.zeros((224, 224, 3), np.uint8))\n",
    "                invalid_images.append(i)\n",
    "                continue\n",
    "            face_img = image[y1:y2,x1:x2,:]\n",
    "            face_images.append(face_img)\n",
    "\n",
    "        _, batch_scores  = fer.predict_multi_emotions(face_images, logits=False)\n",
    "\n",
    "        for invalid_image_idx in invalid_images:\n",
    "            batch_scores[invalid_image_idx] = np.zeros_like(batch_scores[0]) + 0.125\n",
    "\n",
    "        all_scores.extend(batch_scores)\n",
    "        \n",
    "    all_scores = np.array(all_scores)\n",
    "    return all_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:03<00:00,  3.93s/it]\n",
      "100%|██████████| 1/1 [00:00<00:00, 18.62it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 15.57it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 18.16it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 20.97it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 17.65it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 16.87it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 20.30it/s]\n"
     ]
    }
   ],
   "source": [
    "# Calculate emotion scores for basis images\n",
    "\n",
    "emotions = ['anger', 'contempt', 'disgust', 'fear', 'happiness', 'neutral', 'sadness', 'surprise']\n",
    "emotion_scores = dict()\n",
    "for emotion in emotions:\n",
    "    emotion_basis_path = f'quant_experiments/basis_aligned/{emotion}_src.jpg'\n",
    "    emotion_basis_img = np.array(Image.open(emotion_basis_path).resize((224, 224)), dtype=np.uint8) # Resized since emotion network works on 224x224\n",
    "    emotion_scores[emotion] = extract_emotion_scores_from_images(mtcnn, [emotion_basis_img])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:19<00:00,  1.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion anger is 0.37127119302749634\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion contempt is 0.7066561579704285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion disgust is 0.539374589920044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion fear is 0.5413655042648315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:21<00:00,  1.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion happiness is 0.7332206964492798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:21<00:00,  1.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion neutral is 0.7704249024391174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:22<00:00,  1.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion sadness is 0.7776181101799011\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:22<00:00,  1.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method hyperstyle for emotion surprise is 0.12294494360685349\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:21<00:00,  1.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion anger is 0.4007610082626343\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:23<00:00,  1.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion contempt is 0.7124905586242676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:22<00:00,  1.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion disgust is 0.5651103258132935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:22<00:00,  1.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion fear is 0.4103728234767914\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:21<00:00,  1.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion happiness is 0.9611821174621582\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion neutral is 0.5868839025497437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion sadness is 0.6438469886779785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [00:20<00:00,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emotion score (ES) using method pSp for emotion surprise is 0.08573019504547119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Calculate emotion score (es) for each configuration\n",
    "\n",
    "from glob import glob\n",
    "config_dirs = glob(\"quant_experiments/transfer/*/*/\", recursive = True)\n",
    "\n",
    "for config_dir in config_dirs:\n",
    "    inversion_method = config_dir.split('\\\\')[1]\n",
    "    emotion = config_dir.split('\\\\')[2]\n",
    "\n",
    "    if emotion == 'default':\n",
    "        continue\n",
    "    \n",
    "    image_paths = []\n",
    "    image_paths.extend(glob(os.path.join(config_dir, '**', '*.jpg'), recursive=True))\n",
    "\n",
    "    images = []\n",
    "    for image_path in image_paths:\n",
    "        images.append(np.array(Image.open(image_path).resize((224, 224)), dtype=np.uint8))\n",
    "\n",
    "    scores = extract_emotion_scores_from_images(mtcnn, images)\n",
    "    basis_score = emotion_scores[emotion]\n",
    "\n",
    "    dot_products = np.dot(scores, basis_score)\n",
    "    norm_basis = np.linalg.norm(basis_score)\n",
    "    norms_scores = np.linalg.norm(scores, axis=1)\n",
    "    cosine_similarities = dot_products / (norm_basis * norms_scores)\n",
    "    mean_cosine_similarity = np.mean(cosine_similarities)\n",
    "\n",
    "    print(f'Emotion score (ES) using method {inversion_method} for emotion {emotion} is {mean_cosine_similarity}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}