test_ACVAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import os\n",
    "import pyworld\n",
    "import librosa\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from preprocess import *\n",
    "from model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"model_lambda70_f2f3m1m2\"\n",
    "model_dir = \"./model/\" + model_name\n",
    "\n",
    "data_dir = \"./data/voice_data\"\n",
    "voice_dir_list = [\"F2\", \"F3\", \"M1\", \"M2\"]\n",
    "\n",
    "output_dir = \"./converted_voices/result/\" + model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_rate = 16000\n",
    "num_mcep = 36\n",
    "frame_period = 5.0\n",
    "n_frames = 1024 \n",
    "\n",
    "lambda_p = 70\n",
    "lambda_s = 70\n",
    "nb_label = len(voice_dir_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_save(model, model_dir, model_name):\n",
    "    if not os.path.exists(model_dir):\n",
    "        os.makedirs(model_dir)\n",
    "    torch.save(model.state_dict(), os.path.join(model_dir, model_name))\n",
    "    \n",
    "def model_load(model_dir, model_name):\n",
    "    model = ACVAE(nb_label, lambda_p, lambda_s)\n",
    "    if torch.cuda.is_available():\n",
    "        model.load_state_dict(torch.load(os.path.join(model_dir, model_name), map_location='cuda'))\n",
    "    else:\n",
    "        model.load_state_dict(torch.load(os.path.join(model_dir, model_name), map_location='cpu'))\n",
    "    model.to(device)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_all(model):\n",
    "    print(\"Conversion Start.\")\n",
    "    \n",
    "    if not os.path.exists(output_dir):\n",
    "        os.makedirs(output_dir)i\n",
    "    \n",
    "    for s_label in range(nb_label):\n",
    "        \n",
    "        output_label_dir = os.path.join(output_dir, voice_dir_list[s_label])\n",
    "        if not os.path.exists(output_label_dir):\n",
    "            os.makedirs(output_label_dir)\n",
    "    \n",
    "        voice_path_s = os.path.join(data_dir, voice_dir_list[s_label])\n",
    "\n",
    "        count = -1\n",
    "        files = os.listdir(voice_path_s)\n",
    "        for file in files:\n",
    "            if file.count(\"wav\") == 0:\n",
    "                continue\n",
    "\n",
    "            for t_label in range(nb_label):\n",
    "                if (t_label == s_label):\n",
    "                    continue\n",
    "\n",
    "                voice_path_t = os.path.join(data_dir, voice_dir_list[t_label])\n",
    "\n",
    "                wav, _ = librosa.load(os.path.join(voice_path_s, file), sr = sampling_rate, mono = True)\n",
    "                wav = librosa.util.normalize(wav, norm=np.inf, axis=None)\n",
    "                wav = wav_padding(wav = wav, sr = sampling_rate, frame_period = frame_period, multiple = 4)\n",
    "                f0, timeaxis, sp, ap, mc = world_decompose(wav = wav, fs = sampling_rate, frame_period = frame_period)\n",
    "\n",
    "                mc_transposed  = np.array(mc).T\n",
    "\n",
    "                mcep_normalization_params_s = np.load(os.path.join(voice_path_s, \"mcep_\"+voice_dir_list[s_label]+\".npz\"))\n",
    "                mcep_mean_s = mcep_normalization_params_s['mean']\n",
    "                mcep_std_s = mcep_normalization_params_s['std']    \n",
    "                mcep_normalization_params_t = np.load(os.path.join(voice_path_t, \"mcep_\"+voice_dir_list[t_label]+\".npz\"))\n",
    "                mcep_mean_t = mcep_normalization_params_t['mean']\n",
    "                mcep_std_t = mcep_normalization_params_t['std']\n",
    "\n",
    "                mc_norm = (mc_transposed - mcep_mean_s) / mcep_std_s\n",
    "\n",
    "                x = torch.Tensor(mc_norm).view(1, 1, mc_norm.shape[0], mc_norm.shape[1])\n",
    "\n",
    "                label_s_tensor = torch.Tensor(np.array([s_label])).view(1, 1)\n",
    "                label_t_tensor = torch.Tensor(np.array([t_label])).view(1, 1)\n",
    "\n",
    "                x = x.to(device)\n",
    "                label_s_tensor = label_s_tensor.to(device)\n",
    "                label_t_tensor = label_t_tensor.to(device)\n",
    "\n",
    "                mu_enc, logvar_enc = model.encode(x, label_s_tensor)\n",
    "                z_enc = model.reparameterize(mu_enc, logvar_enc)\n",
    "                # x^\n",
    "                mu_dec_t, logvar_dec_t = model.decode(z_enc, label_t_tensor)\n",
    "                z_dec_t = model.reparameterize(mu_dec_t, logvar_dec_t)\n",
    "                if (torch.cuda.is_available()):\n",
    "                    z_dec_t = z_dec_t.data.cpu().numpy().reshape((mc_norm.shape[0], mc_norm.shape[1]))\n",
    "                else:\n",
    "                    z_dec_t = z_dec_t.data.numpy().reshape((mc_norm.shape[0], mc_norm.shape[1]))\n",
    "                # x_\n",
    "                mu_dec_s, logvar_dec_s = model.decode(z_enc, label_s_tensor)\n",
    "                z_dec_s = model.reparameterize(mu_dec_s, logvar_dec_s)\n",
    "                if (torch.cuda.is_available()):\n",
    "                    z_dec_s = z_dec_s.data.cpu().numpy().reshape((mc_norm.shape[0], mc_norm.shape[1]))\n",
    "                else:\n",
    "                    z_dec_s = z_dec_s.data.numpy().reshape((mc_norm.shape[0], mc_norm.shape[1]))\n",
    "\n",
    "                mc_converted_t = z_dec_t * mcep_std_t + mcep_mean_t\n",
    "                mc_converted_t = mc_converted_t.T\n",
    "                mc_converted_t = np.ascontiguousarray(mc_converted_t)\n",
    "                sp_converted_t = world_decode_mc(mc = mc_converted_t, fs = sampling_rate)\n",
    "                mc_converted_s = z_dec_s * mcep_std_s + mcep_mean_s\n",
    "                mc_converted_s = mc_converted_s.T\n",
    "                mc_converted_s = np.ascontiguousarray(mc_converted_s)\n",
    "                sp_converted_s = world_decode_mc(mc = mc_converted_s, fs = sampling_rate)\n",
    "\n",
    "                sp_gained = np.multiply(sp, np.divide(sp_converted_t, sp_converted_s))\n",
    "\n",
    "                logf0s_normalization_params_s = np.load(os.path.join(voice_path_s, \"log_f0_\"+voice_dir_list[s_label]+\".npz\"))\n",
    "                logf0s_mean_s = logf0s_normalization_params_s['mean']\n",
    "                logf0s_std_s = logf0s_normalization_params_s['std']\n",
    "                logf0s_normalization_params_t = np.load(os.path.join(voice_path_t, \"log_f0_\"+voice_dir_list[t_label]+\".npz\"))\n",
    "                logf0s_mean_t = logf0s_normalization_params_t['mean']\n",
    "                logf0s_std_t = logf0s_normalization_params_t['std']\n",
    "\n",
    "                f0_converted = pitch_conversion(f0 = f0, mean_log_src = logf0s_mean_s, std_log_src = logf0s_std_s, mean_log_target = logf0s_mean_t, std_log_target = logf0s_std_t)\n",
    "\n",
    "                wav_transformed = world_speech_synthesis(f0 = f0_converted, sp = sp_gained, ap = ap, fs = sampling_rate, frame_period = frame_period)\n",
    "                librosa.output.write_wav(os.path.join(output_label_dir, voice_dir_list[s_label]+\"_to_\"+voice_dir_list[t_label]+\"_[\"+file+\"].wav\"), wav_transformed, sampling_rate)\n",
    "                wav_source = world_speech_synthesis(f0 = f0_converted, sp = sp, ap = ap, fs = sampling_rate, frame_period = frame_period)\n",
    "                librosa.output.write_wav(os.path.join(output_label_dir, voice_dir_list[s_label]+\"_to_\"+voice_dir_list[t_label]+\"_[\"+file+\"]_nonconv.wav\"), wav_source, sampling_rate)\n",
    "                \n",
    "            count += 1\n",
    "            if (count % 10 == 0):\n",
    "                print(\"{} ({}/{}) : {:.1f} % is done...\".format(voice_dir_list[s_label], str(s_label+1), str(nb_label), count*100/len(files)))\n",
    "    print(\"Finish.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)\n",
    "\n",
    "model = model_load(model_dir, model_name)\n",
    "conv_all(model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}