Merge_splitter_prepare.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Merge_splitter_prepare.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pren1/VAST/blob/master/Merge_splitter_prepare.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a6rSAc0Ldlzs",
        "colab_type": "code",
        "outputId": "6b6f4c06-301e-4955-89a2-c076ff9b501b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        }
      },
      "source": [
        "!rm -rf VAST/\n",
        "!git clone https://github.com/pren1/VAST.git\n",
        "!pip3 install soundfile\n",
        "!pip3 install librosa"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'VAST'...\n",
            "remote: Enumerating objects: 168, done.\u001b[K\n",
            "remote: Counting objects: 100% (168/168), done.\u001b[K\n",
            "remote: Compressing objects: 100% (134/134), done.\u001b[K\n",
            "remote: Total 168 (delta 79), reused 70 (delta 26), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (168/168), 53.55 MiB | 10.09 MiB/s, done.\n",
            "Resolving deltas: 100% (79/79), done.\n",
            "Collecting soundfile\n",
            "  Downloading https://files.pythonhosted.org/packages/eb/f2/3cbbbf3b96fb9fa91582c438b574cff3f45b29c772f94c400e2c99ef5db9/SoundFile-0.10.3.post1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.6/dist-packages (from soundfile) (1.14.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.0->soundfile) (2.20)\n",
            "Installing collected packages: soundfile\n",
            "Successfully installed soundfile-0.10.3.post1\n",
            "Requirement already satisfied: librosa in /usr/local/lib/python3.6/dist-packages (0.6.3)\n",
            "Requirement already satisfied: resampy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.2.2)\n",
            "Requirement already satisfied: numba>=0.38.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.48.0)\n",
            "Requirement already satisfied: decorator>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (4.4.2)\n",
            "Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (2.1.8)\n",
            "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.22.2.post1)\n",
            "Requirement already satisfied: numpy>=1.8.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.18.5)\n",
            "Requirement already satisfied: joblib>=0.12 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.15.1)\n",
            "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.12.0)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.4.1)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.0->librosa) (0.31.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.0->librosa) (47.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3lqBQzSyHZuT",
        "colab_type": "code",
        "outputId": "35324da2-4b5d-4903-cbf6-4ff07b414767",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        }
      },
      "source": [
        "!pip3 install tqdm\n",
        "!pip3 install matplotlib"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.41.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (3.2.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.2.0)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.18.5)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-UP4LuwdqOZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from VAST.util import *\n",
        "import pdb\n",
        "import pprint\n",
        "import numpy as np\n",
        "import IPython\n",
        "import soundfile as sf\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "import os\n",
        "from multiprocessing import Pool, cpu_count"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CTS9V7G_ISgi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import librosa"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n23yBKnpdriA",
        "colab_type": "code",
        "outputId": "c0ac0483-a585-4a71-c71e-52a16bb3a2c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95UCD5Bkhqkh",
        "colab_type": "code",
        "outputId": "7be97222-092b-4fa8-cb30-88be53a8dd8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!cat /proc/cpuinfo | grep processor | wc -l"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_HFhy3wePRh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# debug_name_list = ['宝鐘マリン', '白上フブキ', '天音かなた', '夏色まつり']\n",
        "debug_name_list = ['白上フブキ', '夏色まつり']\n",
        "# read_in_path = \"gdrive/My Drive/NEW_VERDICT/\"\n",
        "\n",
        "read_in_path = \"gdrive/My Drive/3d_verdict/\"\n",
        "# read_in_path = \"pren1_voice_splitter_bucket/3d_verdict/\"\n",
        "# read_in_path = f\"{local_path}我的云端硬盘/3d_verdict/\"\n",
        "\n",
        "# save_path = \"gdrive/My Drive/Free_RAM_VAST/\"\n",
        "\n",
        "# path_1 = save_path + debug_name_list[1] + \"/\"\n",
        "# path_2 = save_path + debug_name_list[3] + \"/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZPr91GgzQFxZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vad_merge(w):\n",
        "    intervals = librosa.effects.split(w, top_db=20)\n",
        "    temp = list()\n",
        "    for s, e in intervals:\n",
        "        temp.append(w[s:e])\n",
        "    return np.concatenate(temp, axis=None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cwhsYSDlxQA",
        "colab_type": "code",
        "outputId": "891a6c76-1466-447e-9821-4bcf85679414",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "# data_list = []\n",
        "# label_list = []\n",
        "data_dict = {}\n",
        "sample_rate = 8000\n",
        "\n",
        "generate_test_data = False\n",
        "\n",
        "for (index, name) in enumerate(debug_name_list):\n",
        "  data_list = []\n",
        "  target_path = read_in_path + name + \"/\"\n",
        "  first_level_walk = [target_path + f.path.split('/')[-1] for f in os.scandir(target_path) if f.is_dir()]\n",
        "  # if generate_test_data:\n",
        "  #   first_level_walk = first_level_walk[:2]\n",
        "  # else:\n",
        "  #   first_level_walk = first_level_walk[2:]\n",
        "  'later, apply the train/test split here.'\n",
        "  for single_folder_path in tqdm(first_level_walk):\n",
        "    all_files = files_in_target_folder(single_folder_path, '.npy')\n",
        "    for single_npy in all_files:\n",
        "      single_file_path = single_folder_path + \"/\" + single_npy\n",
        "      data = load_data_array_from_npy(single_file_path)\n",
        "      # data, _ = librosa.effects.trim(data, top_db=20)\n",
        "      # data = vad_merge(data)\n",
        "      'data length 300s/8000hz'\n",
        "      for data_clip_index in range(0, len(data), sample_rate * 3):\n",
        "        'make sure you get the 1 seconds clipped data'\n",
        "        current_clip = data[data_clip_index:data_clip_index + sample_rate * 3]\n",
        "        if len(current_clip) == sample_rate * 3:\n",
        "          data_list.append(current_clip)\n",
        "          'corrsponding index here review the source label'\n",
        "          # label_list.append(index)\n",
        "  data_dict[name] = np.asarray(data_list)\n",
        "  print(data_dict[name].shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 33/33 [07:08<00:00, 12.98s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(44400, 24000)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 26/26 [05:09<00:00, 11.89s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(30900, 24000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b52zwxPpWaUf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_dict = load_data_array_from_npy(\"gdrive/My Drive/3d_verdict/Static_dict.npy\").item()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5-INp1bSefuU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# def get_all_files(save_path):\n",
        "#   all_files = files_in_target_folder(save_path, '.npy')\n",
        "#   range_index = np.arange(len(all_files)//2)\n",
        "\n",
        "#   data_list = []\n",
        "#   label_list = []\n",
        "#   for ID in tqdm(range_index):\n",
        "#     data = load_data_array_from_npy(save_path + f'data_{ID}.npy')\n",
        "#     label = load_data_array_from_npy(save_path + f'label_{ID}.npy')\n",
        "#     data_list.append(data)\n",
        "#     label_list.extend(label)\n",
        "#   return np.vstack(data_list), label_list  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhL3pz583P4t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_dict = {}\n",
        "# for name in debug_name_list:\n",
        "#   data_dict[name], _ = get_all_files(save_path + name + \"/\")\n",
        "#   save_data_array_as_npy(data_dict[name], f\"gdrive/My Drive/Free_RAM_VAST/four_people_data_dict_{name}\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vrjPmjOf5FZ8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_dict = {}\n",
        "# for name in debug_name_list:\n",
        "#   data_dict[name] = load_data_array_from_npy(f\"gdrive/My Drive/Free_RAM_VAST/four_people_data_dict_{name}.npy\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gwTs02_9Jvh",
        "colab_type": "code",
        "outputId": "70bd4068-1265-4e4f-c5b0-9a801003162c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        }
      },
      "source": [
        "data_dict"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'夏色まつり': array([[-4.38824631e-02, -8.51969719e-02, -5.36992401e-02, ...,\n",
              "         -2.34751584e-04,  1.09055974e-04, -6.16125355e-04],\n",
              "        [-7.08632171e-04, -2.10939208e-04,  6.42982253e-04, ...,\n",
              "          2.83160979e-08,  2.69284879e-08,  2.56153854e-08],\n",
              "        [ 1.75609202e-07,  3.44422489e-07,  3.59909762e-07, ...,\n",
              "         -1.90959666e-02, -5.40862344e-02, -2.18683593e-02],\n",
              "        ...,\n",
              "        [-2.39831536e-07, -5.56177895e-07,  6.21722620e-08, ...,\n",
              "         -2.06556066e-07, -1.88264053e-07,  9.94393048e-08],\n",
              "        [-1.76505424e-08,  5.53303998e-08,  1.55792605e-08, ...,\n",
              "          5.69141498e-08,  2.58523983e-07,  3.36148460e-08],\n",
              "        [ 2.39245992e-07,  5.75013360e-07,  3.98323095e-07, ...,\n",
              "         -1.62030574e-07,  8.91369183e-08, -1.96988097e-07]], dtype=float32),\n",
              " '白上フブキ': array([[-6.0869069e-03,  3.6055092e-02,  7.2832294e-02, ...,\n",
              "          2.7005184e-03,  1.3216899e-03,  1.4454874e-03],\n",
              "        [ 1.8129856e-03, -6.4096792e-04,  3.7678354e-04, ...,\n",
              "         -1.4713822e-02, -2.0579766e-02, -1.6026461e-02],\n",
              "        [-8.8558774e-03, -1.0188833e-02, -1.9027153e-02, ...,\n",
              "          4.7416300e-02,  5.2858643e-02,  5.5866368e-02],\n",
              "        ...,\n",
              "        [ 2.6474828e-03,  3.4800370e-03,  8.3846180e-03, ...,\n",
              "         -5.6170102e-05, -6.8476074e-05, -7.4230622e-05],\n",
              "        [-1.7108451e-04, -2.2258767e-04, -1.2759354e-04, ...,\n",
              "         -2.7547682e-07,  2.7418520e-07, -2.6783414e-07],\n",
              "        [-3.9595214e-08, -7.6312267e-07, -8.8240461e-07, ...,\n",
              "         -6.2224589e-04, -4.2979221e-04, -2.4628252e-04]], dtype=float32)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKmoq54Ke0qT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_1, _ = get_all_files(path_1)\n",
        "# data_2, _ = get_all_files(path_2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sx-fKMDvhAkY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_2.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lLb2tiJTg-EG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_1.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hx0_YFFThGi_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save_data_array_as_npy(data_1, \"gdrive/My Drive/Free_RAM_VAST/白上フブキ.npy\")\n",
        "# save_data_array_as_npy(data_2, \"gdrive/My Drive/Free_RAM_VAST/夏色まつり.npy\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RXyJ50SXo826",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_1 = load_data_array_from_npy(\"gdrive/My Drive/Free_RAM_VAST/白上フブキ.npy\")\n",
        "# data_2 = load_data_array_from_npy(\"gdrive/My Drive/Free_RAM_VAST/夏色まつり.npy\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bZUBjGbh441O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_1 = data_1[:10000, :]\n",
        "# data_2 = data_2[:10000, :]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Ox32aogiBh3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# adapted from Keith Ito's tacotron implementation\n",
        "# https://github.com/keithito/tacotron/blob/master/util/audio.py\n",
        "\n",
        "import librosa\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class Audio():\n",
        "    def __init__(self):\n",
        "        self.ref_level_db = 20.0\n",
        "        self.n_fft = 1200\n",
        "        self.hop_length = 160\n",
        "        self.win_length = 400\n",
        "        self.min_level_db = -100.0\n",
        "\n",
        "    def wav2spec(self, y):\n",
        "        D = self.stft(y)\n",
        "        S = self.amp_to_db(np.abs(D)) - self.ref_level_db\n",
        "        S, D = self.normalize(S), np.angle(D)\n",
        "        S, D = S.T, D.T # to make [time, freq]\n",
        "        return S, D\n",
        "\n",
        "    def spec2wav(self, spectrogram, phase):\n",
        "        spectrogram, phase = spectrogram.T, phase.T\n",
        "        # used during inference only\n",
        "        # spectrogram: enhanced output\n",
        "        # phase: use noisy input's phase, so no GLA is required\n",
        "        S = self.db_to_amp(self.denormalize(spectrogram) + self.ref_level_db)\n",
        "        return self.istft(S, phase)\n",
        "\n",
        "    def stft(self, y):\n",
        "        return librosa.stft(y=y, n_fft=self.n_fft,\n",
        "                            hop_length=self.hop_length,\n",
        "                            win_length=self.win_length)\n",
        "\n",
        "    def istft(self, mag, phase):\n",
        "        stft_matrix = mag * np.exp(1j*phase)\n",
        "        return librosa.istft(stft_matrix,\n",
        "                             hop_length=self.hop_length,\n",
        "                             win_length=self.win_length)\n",
        "\n",
        "    def amp_to_db(self, x):\n",
        "        return 20.0 * np.log10(np.maximum(1e-5, x))\n",
        "\n",
        "    def db_to_amp(self, x):\n",
        "        return np.power(10.0, x * 0.05)\n",
        "\n",
        "    def normalize(self, S):\n",
        "        return np.clip(S / -self.min_level_db, -1.0, 0.0) + 1.0\n",
        "\n",
        "    def denormalize(self, S):\n",
        "        return (np.clip(S, 0.0, 1.0) - 1.0) * -self.min_level_db"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rDlrABojOZn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "audio = Audio()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zg9ygVtL-qhQ",
        "colab_type": "code",
        "outputId": "a8c0d5e8-5bfa-4c2d-a482-5955c4bdd7d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "np.random.uniform(low=0.5, high=3.0)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2.5414397155980826"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IUwGCZ8IjXdw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mix(s1_target, s2):\n",
        "  srate = 8000\n",
        "  # w1, _ = librosa.effects.trim(s1_target, top_db=20)\n",
        "  # w2, _ = librosa.effects.trim(s2, top_db=20)\n",
        "\n",
        "  # w1_ratio = np.random.uniform(low=0.4, high=3.0)\n",
        "  # random.choice([1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0])\n",
        "  # w2_ratio = np.random.uniform(low=0.4, high=3.0)\n",
        "  # random.choice([1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0])\n",
        "  'Before everything, we normalize the w1 and w2'\n",
        "  # s1_target /= np.max(np.abs(s1_target))\n",
        "  # s2 /= np.max(np.abs(s2))\n",
        "\n",
        "  # w1_ratio = np.random.uniform(low=0.1, high=1.4)\n",
        "  # w1_ratio = np.random.uniform(low=0.5, high=1.5)\n",
        "  # w2_ratio = np.random.uniform(low=0.5, high=1.5)\n",
        "\n",
        "  w1 = s1_target * 1.0\n",
        "  w2 = s2 * 1.0\n",
        "  # plt.plot(s1_target, label=\"s1_target\")\n",
        "  # plt.plot(s2, label=\"s2\")\n",
        "  # plt.legend()\n",
        "  # plt.show()\n",
        "  # IPython.display.display(IPython.display.Audio(np.asarray(w1).T, rate=8000))\n",
        "  # IPython.display.display(IPython.display.Audio(np.asarray(w2).T, rate=8000))\n",
        "  # pdb.set_trace()\n",
        "  mixed = w1 + w2\n",
        "  norm = np.max(np.abs(mixed)) * 1.1\n",
        "  w1, w2, mixed = w1/norm, w2/norm, mixed/norm\n",
        "  # w1_norm = np.max(np.abs(w1)) * 1.1\n",
        "  # w2_norm = np.max(np.abs(w2)) * 1.1\n",
        "  \n",
        "  # if norm < 1e-5:\n",
        "  #   print(\"skipped normalization\")\n",
        "  #   norm = 1\n",
        "\n",
        "  # if w1_norm < 1e-5:\n",
        "  #   print(f\"Skip w1: {w1_norm}\")\n",
        "  #   w1_norm = 1\n",
        "\n",
        "  # if w2_norm < 1e-5:\n",
        "  #   print(f\"Skip w2: {w2_norm}\")\n",
        "  #   w2_norm = 1  \n",
        "\n",
        "  # w1, w2, mixed = w1/w1_norm, w2/w2_norm, mixed/norm\n",
        "  # print(np.max(np.abs(mixed)))\n",
        "  'Not that good, but this is a toy'\n",
        "  # plt.plot(w1, color = 'orange', label='w1')\n",
        "  # plt.plot(w2, color = 'cyan', label='w2')\n",
        "  # plt.show()\n",
        "  # IPython.display.display(IPython.display.Audio(np.asarray(w1).T, rate=8000))\n",
        "  # IPython.display.display(IPython.display.Audio(np.asarray(w2).T, rate=8000))\n",
        "  # IPython.display.display(IPython.display.Audio(np.asarray(mixed).T, rate=8000))\n",
        "  # pdb.set_trace()\n",
        "\n",
        "  target_mag, _ = audio.wav2spec(w1)\n",
        "  # print(np.max(w1))\n",
        "  # resulted_wav = audio.spec2wav(target_mag, phase)\n",
        "  # plt.plot(resulted_wav, color = 'orange', label='resulted_wav')\n",
        "  # plt.show()\n",
        "\n",
        "  # plt.plot(w1, color = 'cyan', label='resulted_wav')\n",
        "  # plt.show()\n",
        "  # pdb.set_trace()\n",
        "\n",
        "  mixed_mag, mixed_phase = audio.wav2spec(mixed)\n",
        "\n",
        "  # plt.plot(target_mag)\n",
        "  # plt.title(\"target_mag\")\n",
        "  # plt.show()\n",
        "\n",
        "  # plt.plot(mixed_mag)\n",
        "  # plt.title(\"mixed_mag\")\n",
        "  # plt.show()\n",
        "\n",
        "  # pdb.set_trace()\n",
        "  return target_mag, mixed_mag, mixed_phase"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cy5J3c2WUB7j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "\n",
        "def generate_data_on_the_fly(batch_size, train_dict):\n",
        "  target_mag_list = []\n",
        "  mixed_mag_list = []\n",
        "  mixed_phase_list = []\n",
        "\n",
        "  speaker_label = []\n",
        "  for i in tqdm(range(100)):\n",
        "    'random select two speakers'\n",
        "    spk1, spk2 = random.sample(['夏色まつり', '白上フブキ'], 2)\n",
        "    # spk1 = '夏色まつり'\n",
        "    # spk2 = '白上フブキ'\n",
        "    spk1_source_data = train_dict[spk1]\n",
        "    spk2_source_data = train_dict[spk2]\n",
        "    \n",
        "    choose_speaker = debug_name_list.index(spk1)\n",
        "\n",
        "    spk1_index = np.random.choice(range(len(spk1_source_data)), batch_size//100)\n",
        "    spk1 = spk1_source_data[spk1_index]\n",
        "    spk2_index = np.random.choice(range(len(spk2_source_data)), batch_size//100)\n",
        "    spk2 = spk2_source_data[spk2_index]\n",
        "\n",
        "    for (s1_target, s2) in zip(spk1, spk2):\n",
        "        target_mag, mixed_mag, mixed_phase = mix(s1_target, s2)\n",
        "        target_mag_list.append(target_mag)\n",
        "        mixed_mag_list.append(mixed_mag)\n",
        "        mixed_phase_list.append(mixed_phase)\n",
        "        speaker_label.append(choose_speaker)\n",
        "  return  np.asarray(target_mag_list), np.asarray(mixed_mag_list), np.asarray(mixed_phase_list), np.asarray(speaker_label)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FT851AmVhj1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !rm -rf \"Verdict/\"\n",
        "\n",
        "# target_folder = \"Verdict/\"\n",
        "# os.mkdir(target_folder)\n",
        "\n",
        "# os.mkdir(f\"{target_folder}/cycle_train_case/\")\n",
        "# os.mkdir(f\"{target_folder}/cycle_val_case/\")\n",
        "# os.mkdir(f\"{target_folder}/cycle_test_case/\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULKPugsLok1Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "target_folder = \"gdrive/My Drive/New_train/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "svGJQmyW7IXq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# def train_wrapper(i):\n",
        "#     target_mag_list, mixed_mag_list, mixed_phase_list, speaker_label = generate_data_on_the_fly(10000, data_dict)\n",
        "#     save_data_array_as_npy(target_mag_list, f\"{target_folder}/cycle_train_case/cycle_target_mag_list_{i}\")\n",
        "#     save_data_array_as_npy(mixed_mag_list, f\"{target_folder}/cycle_train_case/cycle_mixed_mag_list_{i}\")\n",
        "#     save_data_array_as_npy(mixed_phase_list, f\"{target_folder}/cycle_train_case/cycle_mixed_phase_list_{i}\")\n",
        "#     save_data_array_as_npy(speaker_label, f\"{target_folder}/cycle_train_case/cycle_speaker_label{i}\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jLhNVOo7Ffk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# cpu_num = 4\n",
        "# if not generate_test_data:\n",
        "#   arr = list(range(60))\n",
        "#   with Pool(cpu_num) as p:\n",
        "#     r = list(tqdm(p.imap(train_wrapper, arr), total=len(arr)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_PvZFHTW63h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generate_test_data = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gd7tx-4IUUll",
        "colab_type": "code",
        "outputId": "33c97e8a-a33e-421d-cd33-6e95e7dc278c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "if not generate_test_data:\n",
        "  for i in range(60):\n",
        "    target_mag_list, mixed_mag_list, mixed_phase_list, speaker_label = generate_data_on_the_fly(3000, data_dict)\n",
        "    save_data_array_as_npy(target_mag_list, f\"{target_folder}/scale_train_case/cycle_target_mag_list_{i}\")\n",
        "    save_data_array_as_npy(mixed_mag_list, f\"{target_folder}/scale_train_case/cycle_mixed_mag_list_{i}\")\n",
        "    save_data_array_as_npy(mixed_phase_list, f\"{target_folder}/scale_train_case/cycle_mixed_phase_list_{i}\")\n",
        "    save_data_array_as_npy(speaker_label, f\"{target_folder}/scale_train_case/cycle_speaker_label{i}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [02:20<00:00,  1.41s/it]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:08<00:00,  1.45it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:08<00:00,  1.47it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.56it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.56it/s]\n",
            "100%|██████████| 100/100 [01:07<00:00,  1.48it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.49it/s]\n",
            "100%|██████████| 100/100 [01:03<00:00,  1.58it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.50it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.52it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.56it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:14<00:00,  1.35it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:07<00:00,  1.49it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.55it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.50it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:04<00:00,  1.54it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.51it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n",
            "100%|██████████| 100/100 [01:06<00:00,  1.50it/s]\n",
            "100%|██████████| 100/100 [01:05<00:00,  1.53it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXHPwbx2NYMB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generate_test_data = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FEEndjY4ZMGo",
        "colab_type": "code",
        "outputId": "108e63f8-0b3f-4d3a-d821-5d6fe27a6dc9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "if generate_test_data:\n",
        "  target_mag_list, mixed_mag_list, mixed_phase_list, speaker_label = generate_data_on_the_fly(1000, data_dict)\n",
        "  save_data_array_as_npy(target_mag_list, f\"{target_folder}/scale_val_case/cycle_target_mag_list\")\n",
        "  save_data_array_as_npy(mixed_mag_list, f\"{target_folder}/scale_val_case/cycle_mixed_mag_list\")\n",
        "  save_data_array_as_npy(mixed_phase_list, f\"{target_folder}/scale_val_case/cycle_mixed_phase_list\")\n",
        "  save_data_array_as_npy(speaker_label, f\"{target_folder}/scale_val_case/cycle_speaker_label\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [00:22<00:00,  4.54it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlSPDi1IMY36",
        "colab_type": "code",
        "outputId": "b60e54dd-17bf-43d2-86fe-71aa6dd45487",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "if generate_test_data:\n",
        "  target_mag_list, mixed_mag_list, mixed_phase_list, speaker_label = generate_data_on_the_fly(1000, data_dict)\n",
        "  save_data_array_as_npy(target_mag_list, f\"{target_folder}/scale_test_case/cycle_target_mag_list\")\n",
        "  save_data_array_as_npy(mixed_mag_list, f\"{target_folder}/scale_test_case/cycle_mixed_mag_list\")\n",
        "  save_data_array_as_npy(mixed_phase_list, f\"{target_folder}/scale_test_case/cycle_mixed_phase_list\")\n",
        "  save_data_array_as_npy(speaker_label, f\"{target_folder}/scale_test_case/cycle_speaker_label\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [00:25<00:00,  3.94it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cLUVdepWj5VF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# target_mag_list = []\n",
        "# mixed_mag_list = []\n",
        "# mixed_phase_list = []\n",
        "\n",
        "# for (s1_target, s2) in tqdm(zip(data_1, data_2)):\n",
        "#   target_mag, mixed_mag, mixed_phase = mix(s1_target, s2)\n",
        "#   target_mag_list.append(target_mag)\n",
        "#   mixed_mag_list.append(mixed_mag)\n",
        "#   mixed_phase_list.append(mixed_phase)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95srxJwHsrGq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# np.asarray(target_mag_list).shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0LXXQ0LTtwma",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save_data_array_as_npy(target_mag_list, \"gdrive/My Drive/Free_RAM_VAST/short_target_mag_list\")\n",
        "# save_data_array_as_npy(mixed_mag_list, \"gdrive/My Drive/Free_RAM_VAST/short_mixed_mag_list\")\n",
        "# save_data_array_as_npy(mixed_phase_list, \"gdrive/My Drive/Free_RAM_VAST/short_mixed_phase_list\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E6Gcy2SYW84C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# target_mag = load_data_array_from_npy(\"gdrive/My Drive/Free_RAM_VAST/target_mag_list.npy\")\n",
        "# mixed_mag = load_data_array_from_npy(\"gdrive/My Drive/Free_RAM_VAST/mixed_mag_list.npy\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gsf4YTY7YOin",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# target_mag.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rTaS58x5X3ng",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# target_path = \"gdrive/My Drive/Spitter_VTB/\"\n",
        "\n",
        "# batch_size = 512\n",
        "# save_index = 0\n",
        "# for i in tqdm(range(0, len(target_mag), batch_size)):\n",
        "#   target_cut = target_mag[i:i+batch_size, :, :]\n",
        "#   mixed_cut = mixed_mag[i:i+batch_size, :, :]\n",
        "#   if len(target_cut) == batch_size:\n",
        "#     save_data_array_as_npy(target_cut, target_path + f\"target_mag/{save_index}\")\n",
        "#     save_data_array_as_npy(mixed_cut, target_path + f\"mixed_mag/{save_index}\")\n",
        "#     save_index += 1"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}