card-embeddings-pure-numpy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Yu-Gi-Oh! card embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Descarga la información de mis otros repositorios de GitHub\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile\n",
    "from pathlib import Path\n",
    "import subprocess\n",
    "\n",
    "temporary_directory = tempfile.mkdtemp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cards_repo_url = \"https://github.com/fferegrino/yu-gi-oh.git\"\n",
    "cards_repo_dir = Path(temporary_directory, \"yu-gi-oh\")\n",
    "\n",
    "subprocess.run([\"git\", \"clone\", \"-q\", \"--single-branch\",\n",
    "                \"--branch\", \"pydata2024\",\n",
    "                \"--depth\", \"1\", cards_repo_url, str(cards_repo_dir)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "decks_repo_url = \"https://github.com/fferegrino/yu-gi-oh-decks.git\"\n",
    "decks_repo_dir = Path(temporary_directory, \"yu-gi-oh-decks\")\n",
    "\n",
    "subprocess.run([\"git\", \"clone\", \"-q\", \"--single-branch\",\n",
    "                \"--branch\", \"pydata2024\",\n",
    "                \"--depth\", \"1\", decks_repo_url, str(decks_repo_dir)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's have a quick look at how the data looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards = []\n",
    "with open(cards_repo_dir / \"data/cards.csv\") as r:\n",
    "    reader = csv.DictReader(r)\n",
    "    for card in reader:\n",
    "        cards.append(card)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cards[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "card_variants = []\n",
    "\n",
    "with open(cards_repo_dir / \"data/cards_variants.csv\") as r:\n",
    "    reader = csv.DictReader(r)\n",
    "    for card_variant in reader:\n",
    "        card_variants.append(card_variant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "card_variants[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decks = []\n",
    "\n",
    "for deck_file in decks_repo_dir.glob(\"data/*.csv\"):\n",
    "    with open(deck_file) as r:\n",
    "        reader = csv.DictReader(r)\n",
    "        for deck in reader:\n",
    "            decks.append(deck)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(decks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decks[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert cards to actual Python lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decks[88191][\"side_deck\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "literal_eval(decks[0][\"main_deck\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deck_properties = [\"main_deck\", \"extra_deck\", \"side_deck\"]\n",
    "\n",
    "deck_cards = []\n",
    "\n",
    "for idx, deck in enumerate(decks):\n",
    "    for prop in deck_properties:\n",
    "        passcodes = []\n",
    "        if prop in deck and deck[prop] and deck[prop] != 'null':\n",
    "            try:\n",
    "                deck_passcodes = literal_eval(deck[prop])\n",
    "                passcodes.extend(deck_passcodes)\n",
    "            except:\n",
    "                break\n",
    "        if passcodes:\n",
    "            deck_cards.append({\n",
    "                \"deck_id\": idx,\n",
    "                \"passcodes\": passcodes,\n",
    "            })\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deck_cards[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparación de los diccionarios auxiliares"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "variant_to_original_passcode = {}\n",
    "for variant in card_variants:\n",
    "    variant_to_original_passcode[variant[\"variant\"]] = variant[\"original\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "passcode_to_id = {}\n",
    "id_to_passcode = {}\n",
    "for idx, card in enumerate(cards):\n",
    "    original_passcode = variant_to_original_passcode.get(card[\"id\"], card[\"id\"])\n",
    "    passcode_to_id[original_passcode] = idx\n",
    "    id_to_passcode[str(idx)] = original_passcode\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(\"passcode_variants.json\", \"w\") as w:\n",
    "    json.dump(variant_to_original_passcode, w)\n",
    "\n",
    "with open(\"passcode_to_id.json\", \"w\") as w:\n",
    "    json.dump(passcode_to_id, w, indent=4)\n",
    "\n",
    "with open(\"id_to_passcode.json\", \"w\") as w:\n",
    "    json.dump(id_to_passcode, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_card_id(passcode):\n",
    "    original_passcode = variant_to_original_passcode.get(passcode, passcode)\n",
    "    return passcode_to_id.get(original_passcode, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_card_id(\"36996508\"), get_card_id(\"46986414\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generación de la matriz de co-ocurrencia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "card_count = len(cards)\n",
    "\n",
    "# Initialize a dense matrix with zeros\n",
    "matrix = np.zeros((card_count, card_count), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from itertools import permutations\n",
    "\n",
    "for card in deck_cards[:1000]:\n",
    "    card_permutation_count = Counter(permutations(card[\"passcodes\"], 2))\n",
    "    for (card1, card2), count in card_permutation_count.items():\n",
    "        card1_id = get_card_id(card1)\n",
    "        card2_id = get_card_id(card2)\n",
    "        if card1_id is not None and card2_id is not None:\n",
    "            matrix[card1_id, card2_id] += count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cálculo de los embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the full SVD\n",
    "u, s, _ = np.linalg.svd(matrix, full_matrices=False)\n",
    "\n",
    "# Select the first `k` singular values/vectors\n",
    "k = embedding_size\n",
    "u_k = u[:, :k]\n",
    "s_k = s[:k]\n",
    "\n",
    "# Compute the embeddings\n",
    "embeddings = u_k * np.sqrt(s_k)\n",
    "\n",
    "# Normalize the embeddings\n",
    "embeddings = (embeddings - np.mean(embeddings, axis=0)) / np.std(embeddings, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_file = \"card-embeddings.ann\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from annoy import AnnoyIndex\n",
    "\n",
    "ann = AnnoyIndex(embedding_size, \"angular\")\n",
    "number_of_trees = 10\n",
    "\n",
    "for idx, card_vector in enumerate(embeddings):\n",
    "    ann.add_item(idx, card_vector)\n",
    "\n",
    "ann.build(number_of_trees)\n",
    "ann.save(index_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ejecutando algunos queries en el índice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import requests\n",
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "\n",
    "def index_to_card(idx):\n",
    "    passcode = id_to_passcode[str(idx)]\n",
    "    card = next((card for card in cards if card[\"id\"] == passcode), None)\n",
    "    return card\n",
    "\n",
    "def passcode_to_card(passcode):\n",
    "    original_passcode = variant_to_original_passcode.get(passcode, passcode)\n",
    "    card = next((card for card in cards if card[\"id\"] == original_passcode), None)\n",
    "    return card\n",
    "\n",
    "def draw_cards(cards):\n",
    "    fig, axes = plt.subplots(nrows=1, ncols=len(cards), figsize=(20, 4))  # Adjust figsize as needed\n",
    "\n",
    "    for i, card in enumerate(cards):\n",
    "        response = requests.get(card[\"image_url_small\"])\n",
    "        if response.status_code == 200:\n",
    "            img = Image.open(BytesIO(response.content))\n",
    "            axes[i].imshow(img)\n",
    "            axes[i].axis('off')\n",
    "        else:\n",
    "            print(f\"Failed to load image  for {card['name']}\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from annoy import AnnoyIndex\n",
    "\n",
    "index_file = \"card-embeddings.ann\"\n",
    "embedding_size = 50\n",
    "\n",
    "ann = AnnoyIndex(embedding_size, \"angular\")\n",
    "ann.load(index_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_card = '46986414' # Dark Magician\n",
    "# query_card = '14558127' # Ash Blossom & Joyous Spring\n",
    "card = passcode_to_card(query_card)\n",
    "query_card_embedding = ann.get_item_vector(get_card_id(query_card))\n",
    "print(query_card_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similar_card_ids = ann.get_nns_by_vector(query_card_embedding, 5)\n",
    "print(similar_card_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similar_cards = [index_to_card(idx) for idx in similar_card_ids]\n",
    "draw_cards(similar_cards)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}