#2-train_vae.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "AG4vz6r8ZB80"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Input, Conv2D, BatchNormalization,LeakyReLU, ReLU, Flatten,Dense, Reshape, Conv2DTranspose, Activation, Lambda \n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.losses import MeanSquaredError\n",
    "import numpy as np\n",
    "import os \n",
    "import pickle\n",
    "\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "id": "gtc2IZkZbfKJ"
   },
   "outputs": [],
   "source": [
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "id": "WQBSrae5bhYk"
   },
   "outputs": [],
   "source": [
    "class VAE :\n",
    "    #constructor\n",
    "    def  __init__(self, input_shape, conv_filters, \n",
    "                  conv_kernels, conv_strides,\n",
    "                  latent_space_dim):\n",
    "        \n",
    "        self.input_shape = input_shape #for images we have w*h* colors [28,28,1]\n",
    "        self.conv_filters = conv_filters #list of number of filters for conv layers [2,4,8]\n",
    "        self.conv_kernels = conv_kernels #kernal sizes for each layer [3,5,3]\n",
    "        self.conv_strides = conv_strides #strides for each encoder [1,2,2]\n",
    "        self.latent_space_dim = latent_space_dim #2\n",
    "        \n",
    "        self.encoder = None #encoder model\n",
    "        self.decoder= None\n",
    "        self.model = None # the total model\n",
    "        self.reconstruction_loss_weight = 1000000\n",
    "        \n",
    "        \n",
    "        #private values\n",
    "        self._num_conv_layers = len(conv_filters)\n",
    "        self._shape_before_bottleneck = None\n",
    "        self._model_input = None\n",
    "        self._build()\n",
    "        \n",
    "    def summary(self):\n",
    "        self.encoder.summary()\n",
    "        self.decoder.summary()\n",
    "        self.model.summary()\n",
    "\n",
    "    def _build(self):\n",
    "        self._build_encoder()\n",
    "        self._build_decoder()\n",
    "        self._build_autoencoder()\n",
    "        \n",
    "    def _build_encoder(self) :\n",
    "        #creating input\n",
    "        encoder_input = self._add_encoder_input()\n",
    "        #build conv layers\n",
    "        conv_layers = self._add_conv_layers(encoder_input)\n",
    "        #build bottleneck\n",
    "        bottleneck = self.add_bottleneck(conv_layers)\n",
    "        self._model_input = encoder_input  \n",
    "        \n",
    "        self.encoder = Model(encoder_input, bottleneck, name = \"encoder\")\n",
    "    \n",
    "    def _add_encoder_input(self) :\n",
    "        #build input layer with specific shape\n",
    "        return Input(shape = self.input_shape, name = \"encoder_input\")\n",
    "    \n",
    "    def _add_conv_layers(self, encoder_input) :\n",
    "        x = encoder_input\n",
    "        for layer_index in range(self._num_conv_layers):\n",
    "            x= self._add_conv_layer(layer_index,x)\n",
    "        return x\n",
    "    \n",
    "    def _add_conv_layer(self, layer_index,x ) :\n",
    "        layer_number = layer_index + 1 \n",
    "        conv_layer = Conv2D(\n",
    "            filters=self.conv_filters[layer_index],\n",
    "            kernel_size=self.conv_kernels[layer_index],\n",
    "            strides=self.conv_strides[layer_index],\n",
    "            padding=\"same\",\n",
    "            name=f\"enocder_conve_layer_{layer_index}\"\n",
    "        )\n",
    "        x = conv_layer(x)\n",
    "        x = ReLU(name = f\"encoder_relu{layer_number}\")(x)\n",
    "        x= BatchNormalization(name = f\"encoder_bn{layer_number}\")(x)\n",
    "        return x\n",
    "    \n",
    "    def add_bottleneck(self, x) :\n",
    "        \"\"\" flatten the conv2d and then pass to dense layer\"\"\"\n",
    "        self._shape_before_bottleneck = K.int_shape(x)[1:] #to be mirrored in decoder\n",
    "        x = Flatten()(x)\n",
    "\n",
    "        #the change here is instead of latent space fixed values, we will output two\n",
    "        # neural network one for MUs and one for Variance\n",
    "        #  NOT SEQUN\n",
    "        self.mu = Dense(self.latent_space_dim ,name = \"mu\")(x)\n",
    "        self.log_variance = Dense(self.latent_space_dim, name = \"log_variance\")(x)\n",
    "\n",
    "        #the output should be one network to be sampled, and we need to sampe from the previous\n",
    "        # two networks\n",
    "        # this can be done using lambda layer\n",
    "\n",
    "        # Lambda is used to transform the input data using an expression or function.\n",
    "        #   For example, if Lambda with expression lambda x: x ** 2 is applied to a layer,\n",
    "        #   then its input data will be squared before processing.\n",
    "        \n",
    "        #it will sample point from ND and output it\n",
    "        def sample_point_from_normal_distribution(args):\n",
    "          mu, log_variance = args\n",
    "          #sampling random point from normal dist with mu =0 and dev = 1.0 i.e. standard ND\n",
    "          epsillon = K.random_normal(shape = K.shape(self.mu), mean= 0.0, stddev = 1.0)\n",
    "\n",
    "          sampled_point  = mu + K.exp(log_variance/2)* epsillon \n",
    "          return sampled_point\n",
    "\n",
    "\n",
    "\n",
    "        encoder_output = Lambda(sample_point_from_normal_distribution,\n",
    "                   name = \"encoder_output\")([self.mu, self.log_variance])\n",
    "        return encoder_output\n",
    "    \n",
    "    \n",
    "    \n",
    "    def _build_decoder(self):\n",
    "        #creating input\n",
    "        decoder_input = self._add_decoder_input() #latent dim 2*2\n",
    "        dense_layer= self._add_dense_layer(decoder_input) #connect\n",
    "        reshape_layer = self._add_reshape_layer(dense_layer) #reshape to dense\n",
    "        conv_transpose_layers = self._add_conv_transpose_layers(reshape_layer)#de conv\n",
    "        decoder_output = self._add_decoder_output(conv_transpose_layers)\n",
    "        self.decoder  = Model(decoder_input ,decoder_output , name = \"decoder\")\n",
    "    \n",
    "    def _add_decoder_input(self):\n",
    "        return Input(shape= self.latent_space_dim, name= \"decoder_input\")\n",
    "    \n",
    "    def _add_dense_layer(self, decoder_input):\n",
    "        num_neurons =np.prod(self._shape_before_bottleneck)\n",
    "        dense_layer = Dense(num_neurons, name = \"decoder_dense\")(decoder_input)\n",
    "        return dense_layer\n",
    "\n",
    "    def _add_reshape_layer(self, dense_layer):\n",
    "        #convert the flat to original shape\n",
    "        reshape_layer = Reshape(self._shape_before_bottleneck, name = \"reshape_layer\")(dense_layer)\n",
    "        return reshape_layer\n",
    "    \n",
    "    def _add_conv_transpose_layers(self, x ) :\n",
    "        #decode\n",
    "        for layer_index in reversed(range(1, self._num_conv_layers)):\n",
    "            x = self._add_conv_transpose_layer(layer_index, x)\n",
    "        return x\n",
    "    \n",
    "    def _add_conv_transpose_layer(self,layer_index, x ) :\n",
    "        layer_num = self._num_conv_layers - layer_index\n",
    "        conv_transpose_layer = Conv2DTranspose(\n",
    "            filters = self.conv_filters[layer_index],\n",
    "            kernel_size = self.conv_kernels[layer_index],\n",
    "            strides = self.conv_strides[layer_index],\n",
    "            padding = \"same\",\n",
    "            name = f\"decodeer_deconv_layer_{layer_num}\"\n",
    "        )\n",
    "        x = conv_transpose_layer(x)\n",
    "        x= ReLU(name = f\"decodeer_ReLu_layer_{layer_num}\")(x)\n",
    "        x = BatchNormalization(name = f\"decodeer_BN_layer_{layer_num}\")(x)\n",
    "        return x\n",
    "    \n",
    "    \n",
    "    def _add_decoder_output(self,conv_transpose_layer ) :\n",
    "        last_conv_transpose_layer = Conv2DTranspose(\n",
    "            filters = 1, #set to BW image\n",
    "            kernel_size = self.conv_kernels[0],\n",
    "            strides = self.conv_strides[0],\n",
    "            padding = \"same\",\n",
    "            name = f\"decodeer_deconv_layer_{self._num_conv_layers}\"\n",
    "        )\n",
    "        conv_transpose_layer = last_conv_transpose_layer(conv_transpose_layer)\n",
    "        output_layer = Activation(\"sigmoid\", name = \"sigmoid\")(conv_transpose_layer)\n",
    "        return output_layer\n",
    "    \n",
    "    def _build_autoencoder(self) :\n",
    "        model_input  = self._model_input \n",
    "        model_output = self.decoder(self.encoder(model_input))\n",
    "        self.model = Model(model_input,model_output, name=\"autoencoder\" )\n",
    "        \n",
    "    def compile(self, learning_rate = 0.0001):\n",
    "        #optimizer\n",
    "        optimizer = Adam(learning_rate= learning_rate)\n",
    "        self.model.compile(optimizer =optimizer , loss = self._calculate_combianed_loss,\n",
    "                           metrics= [self._calculate_reconstruction_loss, self._calculate_kl_loss] \n",
    "                           ) \n",
    "        \n",
    "    def train(self, x_train, batch_size, num_epochs, initial_epoch,  ):\n",
    "        \n",
    "\n",
    "        checkpoint_filepath=os.path.join('.', \"weights\\weights-{epoch:03d}-{loss:.2f}.h5\")\n",
    "        checkpoint1 = ModelCheckpoint(checkpoint_filepath, save_weights_only = True, verbose=1)\n",
    "        checkpoint2 = ModelCheckpoint(os.path.join('.', 'weights\\weights.h5'), save_weights_only = True, verbose=1)\n",
    "        \n",
    "        callbacks_list = [checkpoint1, checkpoint2] # custom_callback]\n",
    "        \n",
    "        self.model.fit(x_train, x_train, batch_size=batch_size, shuffle = True, epochs =num_epochs, initial_epoch = initial_epoch\n",
    "            , callbacks = callbacks_list  )\n",
    "                      \n",
    "    def save(self , save_folder=\".\"):\n",
    "        self._create_folder_if_not_exist(save_folder)\n",
    "        self._save_paramters(save_folder)\n",
    "        self._save_weights(save_folder)\n",
    "\n",
    "    def reconstruct(self, images):\n",
    "        latent_representations = self.encoder.predict(images)\n",
    "        reconstructed_images = self.decoder.predict(latent_representations)\n",
    "        return reconstructed_images, latent_representations\n",
    "    \n",
    "    @classmethod\n",
    "    def load(cls, save_folder=\".\"):\n",
    "        parameters_path = os.path.join(save_folder, \"parameters.pkl\")\n",
    "        with open(parameters_path, \"rb\") as f:\n",
    "            parameters = pickle.load(f)\n",
    "        autoencoder = VAE(*parameters)\n",
    "        weights_path = os.path.join(save_folder, \"weights.h5\")\n",
    "        autoencoder.load_weights(weights_path)\n",
    "        return autoencoder\n",
    "    \n",
    "    def load_weights(self, weights_path):\n",
    "        self.model.load_weights(weights_path)\n",
    "\n",
    "    def _calculate_combianed_loss(self, y_target, y_predicted) :\n",
    "      reconstruction_loss = self._calculate_reconstruction_loss( y_target, y_predicted)\n",
    "      kl_loss = self._calculate_kl_loss(y_target, y_predicted)\n",
    "      combined_loss = self.reconstruction_loss_weight * reconstruction_loss+ kl_loss\n",
    "      return combined_loss\n",
    "\n",
    "    def _calculate_reconstruction_loss(self, y_target, y_predicted):\n",
    "      error = y_target - y_predicted\n",
    "      reconstruction_loss = K.mean(K.square(error), axis=[1, 2, 3])\n",
    "      return reconstruction_loss\n",
    "\n",
    "    def _calculate_kl_loss(self, y_target, y_predicted):\n",
    "        kl_loss = -0.5 * K.sum(1 + self.log_variance - K.square(self.mu) -\n",
    "                               K.exp(self.log_variance), axis=1)\n",
    "        return kl_loss\n",
    "\n",
    "    def _create_folder_if_not_exist(self ,save_folder):\n",
    "        if not os.path.exists(save_folder):\n",
    "            os.makedirs(save_folder)\n",
    "    \n",
    "    def _save_paramters(self, save_folder):\n",
    "        paramters = [\n",
    "            self.input_shape,\n",
    "            self.conv_filters,\n",
    "            self.conv_kernels,\n",
    "            self.conv_strides,\n",
    "            self.latent_space_dim\n",
    "        ]\n",
    "        save_path = os.path.join(save_folder, \"paramters.pkl\")\n",
    "        with open(save_path , \"wb\") as f:\n",
    "            pickle.dump(paramters, f)\n",
    "        \n",
    "    def _save_weights(self, save_folder):\n",
    "        save_path = os.path.join(save_folder, \"weights.h5\")\n",
    "        self.model.save_weights(save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "KojuIQI8iW3I"
   },
   "outputs": [],
   "source": [
    "LR = 0.0005\n",
    "EPOCHS = 250\n",
    "BS = 64\n",
    "\n",
    "SPECTROGRAMS_PATH = \"./spectrograms\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "BrCwS38ncNQK"
   },
   "outputs": [],
   "source": [
    "def load_fsdd(spectrogram_path):\n",
    "  x_train = []\n",
    "  for root, subroot, file_names in os.walk(spectrogram_path):\n",
    "    for file_name in file_names :\n",
    "      file_path = os.path.join(root, file_name) \n",
    "      spectogram = np.load(file_path) # this will return (n_bins, n_frames) but for VAE it needs three dimensions \n",
    "      x_train.append(spectogram)\n",
    "  x_train = np.array(x_train)\n",
    "  x_train = x_train[..., np.newaxis] #(n_samples, n_bins, n_frames, 1)\n",
    "  #VAE requires W*H*C here we will set channel number(C) to 1\n",
    "  return x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "U9yTgwvrf7_T"
   },
   "outputs": [],
   "source": [
    "def train(x_train, learning_rate, batch_size, epochs, inital_epoch):\n",
    "    autoencoder = VAE(\n",
    "        input_shape=(256, 64, 1),\n",
    "        conv_filters=(512, 256, 128, 64, 32),\n",
    "        conv_kernels=(3, 3, 3, 3, 3),\n",
    "        conv_strides=(2, 2, 2, 2, (2, 1)),\n",
    "        latent_space_dim=128\n",
    "    )\n",
    "    autoencoder.summary()\n",
    "    autoencoder.compile(learning_rate)\n",
    "    autoencoder.train(x_train, batch_size, epochs, inital_epoch)\n",
    "    return autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BUYRpgEgbjiK",
    "outputId": "4f031530-072a-4b98-88c9-e5ee38905e51"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"encoder\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "encoder_input (InputLayer)      [(None, 256, 64, 1)] 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "enocder_conve_layer_0 (Conv2D)  (None, 128, 32, 512) 5120        encoder_input[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "encoder_relu1 (ReLU)            (None, 128, 32, 512) 0           enocder_conve_layer_0[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "encoder_bn1 (BatchNormalization (None, 128, 32, 512) 2048        encoder_relu1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "enocder_conve_layer_1 (Conv2D)  (None, 64, 16, 256)  1179904     encoder_bn1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "encoder_relu2 (ReLU)            (None, 64, 16, 256)  0           enocder_conve_layer_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "encoder_bn2 (BatchNormalization (None, 64, 16, 256)  1024        encoder_relu2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "enocder_conve_layer_2 (Conv2D)  (None, 32, 8, 128)   295040      encoder_bn2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "encoder_relu3 (ReLU)            (None, 32, 8, 128)   0           enocder_conve_layer_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "encoder_bn3 (BatchNormalization (None, 32, 8, 128)   512         encoder_relu3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "enocder_conve_layer_3 (Conv2D)  (None, 16, 4, 64)    73792       encoder_bn3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "encoder_relu4 (ReLU)            (None, 16, 4, 64)    0           enocder_conve_layer_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "encoder_bn4 (BatchNormalization (None, 16, 4, 64)    256         encoder_relu4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "enocder_conve_layer_4 (Conv2D)  (None, 8, 4, 32)     18464       encoder_bn4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "encoder_relu5 (ReLU)            (None, 8, 4, 32)     0           enocder_conve_layer_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "encoder_bn5 (BatchNormalization (None, 8, 4, 32)     128         encoder_relu5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)            (None, 1024)         0           encoder_bn5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "mu (Dense)                      (None, 128)          131200      flatten_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "log_variance (Dense)            (None, 128)          131200      flatten_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "encoder_output (Lambda)         (None, 128)          0           mu[0][0]                         \n",
      "                                                                 log_variance[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 1,838,688\n",
      "Trainable params: 1,836,704\n",
      "Non-trainable params: 1,984\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"decoder\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "decoder_input (InputLayer)   [(None, 128)]             0         \n",
      "_________________________________________________________________\n",
      "decoder_dense (Dense)        (None, 1024)              132096    \n",
      "_________________________________________________________________\n",
      "reshape_layer (Reshape)      (None, 8, 4, 32)          0         \n",
      "_________________________________________________________________\n",
      "decodeer_deconv_layer_1 (Con (None, 16, 4, 32)         9248      \n",
      "_________________________________________________________________\n",
      "decodeer_ReLu_layer_1 (ReLU) (None, 16, 4, 32)         0         \n",
      "_________________________________________________________________\n",
      "decodeer_BN_layer_1 (BatchNo (None, 16, 4, 32)         128       \n",
      "_________________________________________________________________\n",
      "decodeer_deconv_layer_2 (Con (None, 32, 8, 64)         18496     \n",
      "_________________________________________________________________\n",
      "decodeer_ReLu_layer_2 (ReLU) (None, 32, 8, 64)         0         \n",
      "_________________________________________________________________\n",
      "decodeer_BN_layer_2 (BatchNo (None, 32, 8, 64)         256       \n",
      "_________________________________________________________________\n",
      "decodeer_deconv_layer_3 (Con (None, 64, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "decodeer_ReLu_layer_3 (ReLU) (None, 64, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "decodeer_BN_layer_3 (BatchNo (None, 64, 16, 128)       512       \n",
      "_________________________________________________________________\n",
      "decodeer_deconv_layer_4 (Con (None, 128, 32, 256)      295168    \n",
      "_________________________________________________________________\n",
      "decodeer_ReLu_layer_4 (ReLU) (None, 128, 32, 256)      0         \n",
      "_________________________________________________________________\n",
      "decodeer_BN_layer_4 (BatchNo (None, 128, 32, 256)      1024      \n",
      "_________________________________________________________________\n",
      "decodeer_deconv_layer_5 (Con (None, 256, 64, 1)        2305      \n",
      "_________________________________________________________________\n",
      "sigmoid (Activation)         (None, 256, 64, 1)        0         \n",
      "=================================================================\n",
      "Total params: 533,089\n",
      "Trainable params: 532,129\n",
      "Non-trainable params: 960\n",
      "_________________________________________________________________\n",
      "Model: \"autoencoder\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "encoder_input (InputLayer)   [(None, 256, 64, 1)]      0         \n",
      "_________________________________________________________________\n",
      "encoder (Model)              (None, 128)               1838688   \n",
      "_________________________________________________________________\n",
      "decoder (Model)              (None, 256, 64, 1)        533089    \n",
      "=================================================================\n",
      "Total params: 2,371,777\n",
      "Trainable params: 2,368,833\n",
      "Non-trainable params: 2,944\n",
      "_________________________________________________________________\n",
      "Epoch 1/250\n",
      "2944/3000 [============================>.] - ETA: 3:26 - loss: 127718.4447 - _calculate_reconstruction_loss: 0.1262 - _calculate_kl_loss: 1506.7189\n",
      "Epoch 00001: saving model to .\\weights\\weights-001-126183.96.h5\n",
      "\n",
      "Epoch 00001: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 10916s 4s/sample - loss: 126183.9582 - _calculate_reconstruction_loss: 0.1247 - _calculate_kl_loss: 1495.5229\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 33273.3336 - _calculate_reconstruction_loss: 0.0326 - _calculate_kl_loss: 639.1792 \n",
      "Epoch 00002: saving model to .\\weights\\weights-002-33099.68.h5\n",
      "\n",
      "Epoch 00002: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2259s 753ms/sample - loss: 33099.6822 - _calculate_reconstruction_loss: 0.0325 - _calculate_kl_loss: 634.5101\n",
      "Epoch 3/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 22679.2801 - _calculate_reconstruction_loss: 0.0223 - _calculate_kl_loss: 358.7851 \n",
      "Epoch 00003: saving model to .\\weights\\weights-003-22602.78.h5\n",
      "\n",
      "Epoch 00003: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2294s 765ms/sample - loss: 22602.7755 - _calculate_reconstruction_loss: 0.0222 - _calculate_kl_loss: 358.5272\n",
      "Epoch 4/250\n",
      "2944/3000 [============================>.] - ETA: 43s - loss: 18091.2338 - _calculate_reconstruction_loss: 0.0178 - _calculate_kl_loss: 297.2385 \n",
      "Epoch 00004: saving model to .\\weights\\weights-004-18071.00.h5\n",
      "\n",
      "Epoch 00004: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2312s 771ms/sample - loss: 18070.9993 - _calculate_reconstruction_loss: 0.0178 - _calculate_kl_loss: 296.8854\n",
      "Epoch 5/250\n",
      "2944/3000 [============================>.] - ETA: 43s - loss: 15644.1284 - _calculate_reconstruction_loss: 0.0154 - _calculate_kl_loss: 268.1806 \n",
      "Epoch 00005: saving model to .\\weights\\weights-005-15604.79.h5\n",
      "\n",
      "Epoch 00005: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2307s 769ms/sample - loss: 15604.7930 - _calculate_reconstruction_loss: 0.0153 - _calculate_kl_loss: 268.6168\n",
      "Epoch 6/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 11647.1805 - _calculate_reconstruction_loss: 0.0114 - _calculate_kl_loss: 277.0579 \n",
      "Epoch 00006: saving model to .\\weights\\weights-006-11640.36.h5\n",
      "\n",
      "Epoch 00006: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2300s 767ms/sample - loss: 11640.3585 - _calculate_reconstruction_loss: 0.0114 - _calculate_kl_loss: 276.6969\n",
      "Epoch 7/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 10868.8726 - _calculate_reconstruction_loss: 0.0106 - _calculate_kl_loss: 249.8596 \n",
      "Epoch 00007: saving model to .\\weights\\weights-007-10846.56.h5\n",
      "\n",
      "Epoch 00007: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2294s 765ms/sample - loss: 10846.5635 - _calculate_reconstruction_loss: 0.0106 - _calculate_kl_loss: 249.7991\n",
      "Epoch 8/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 10558.4158 - _calculate_reconstruction_loss: 0.0103 - _calculate_kl_loss: 241.3006 \n",
      "Epoch 00008: saving model to .\\weights\\weights-008-10557.21.h5\n",
      "\n",
      "Epoch 00008: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2287s 762ms/sample - loss: 10557.2056 - _calculate_reconstruction_loss: 0.0103 - _calculate_kl_loss: 241.3007\n",
      "Epoch 9/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 10266.9755 - _calculate_reconstruction_loss: 0.0100 - _calculate_kl_loss: 234.4042 \n",
      "Epoch 00009: saving model to .\\weights\\weights-009-10268.67.h5\n",
      "\n",
      "Epoch 00009: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2296s 765ms/sample - loss: 10268.6698 - _calculate_reconstruction_loss: 0.0100 - _calculate_kl_loss: 234.3233\n",
      "Epoch 10/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 10275.5466 - _calculate_reconstruction_loss: 0.0100 - _calculate_kl_loss: 232.9395 \n",
      "Epoch 00010: saving model to .\\weights\\weights-010-10269.84.h5\n",
      "\n",
      "Epoch 00010: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2299s 766ms/sample - loss: 10269.8427 - _calculate_reconstruction_loss: 0.0100 - _calculate_kl_loss: 232.9038\n",
      "Epoch 11/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 10037.8931 - _calculate_reconstruction_loss: 0.0098 - _calculate_kl_loss: 227.1850 \n",
      "Epoch 00011: saving model to .\\weights\\weights-011-10036.06.h5\n",
      "\n",
      "Epoch 00011: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2297s 766ms/sample - loss: 10036.0611 - _calculate_reconstruction_loss: 0.0098 - _calculate_kl_loss: 227.1985\n",
      "Epoch 12/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9910.1635 - _calculate_reconstruction_loss: 0.0097 - _calculate_kl_loss: 226.9270 \n",
      "Epoch 00012: saving model to .\\weights\\weights-012-9894.67.h5\n",
      "\n",
      "Epoch 00012: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2288s 763ms/sample - loss: 9894.6654 - _calculate_reconstruction_loss: 0.0097 - _calculate_kl_loss: 226.9794\n",
      "Epoch 13/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9790.2845 - _calculate_reconstruction_loss: 0.0096 - _calculate_kl_loss: 227.3186 \n",
      "Epoch 00013: saving model to .\\weights\\weights-013-9780.46.h5\n",
      "\n",
      "Epoch 00013: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2288s 763ms/sample - loss: 9780.4642 - _calculate_reconstruction_loss: 0.0096 - _calculate_kl_loss: 227.2935\n",
      "Epoch 14/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9728.9093 - _calculate_reconstruction_loss: 0.0095 - _calculate_kl_loss: 225.9817 \n",
      "Epoch 00014: saving model to .\\weights\\weights-014-9724.08.h5\n",
      "\n",
      "Epoch 00014: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2295s 765ms/sample - loss: 9724.0751 - _calculate_reconstruction_loss: 0.0095 - _calculate_kl_loss: 225.9477\n",
      "Epoch 15/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9688.4011 - _calculate_reconstruction_loss: 0.0095 - _calculate_kl_loss: 224.0022 \n",
      "Epoch 00015: saving model to .\\weights\\weights-015-9673.11.h5\n",
      "\n",
      "Epoch 00015: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2294s 765ms/sample - loss: 9673.1129 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 224.0436\n",
      "Epoch 16/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9650.6750 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 223.3073 \n",
      "Epoch 00016: saving model to .\\weights\\weights-016-9644.77.h5\n",
      "\n",
      "Epoch 00016: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2285s 762ms/sample - loss: 9644.7737 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 223.2021\n",
      "Epoch 17/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9588.6795 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 220.5083 \n",
      "Epoch 00017: saving model to .\\weights\\weights-017-9584.58.h5\n",
      "\n",
      "Epoch 00017: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2291s 764ms/sample - loss: 9584.5834 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 220.4856\n",
      "Epoch 18/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9570.8102 - _calculate_reconstruction_loss: 0.0094 - _calculate_kl_loss: 220.7495 \n",
      "Epoch 00018: saving model to .\\weights\\weights-018-9561.71.h5\n",
      "\n",
      "Epoch 00018: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2294s 765ms/sample - loss: 9561.7147 - _calculate_reconstruction_loss: 0.0093 - _calculate_kl_loss: 220.6983\n",
      "Epoch 19/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9527.8079 - _calculate_reconstruction_loss: 0.0093 - _calculate_kl_loss: 220.2548 \n",
      "Epoch 00019: saving model to .\\weights\\weights-019-9533.93.h5\n",
      "\n",
      "Epoch 00019: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2295s 765ms/sample - loss: 9533.9331 - _calculate_reconstruction_loss: 0.0093 - _calculate_kl_loss: 220.2570\n",
      "Epoch 20/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9548.8255 - _calculate_reconstruction_loss: 0.0093 - _calculate_kl_loss: 218.3902 \n",
      "Epoch 00020: saving model to .\\weights\\weights-020-9553.00.h5\n",
      "\n",
      "Epoch 00020: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2289s 763ms/sample - loss: 9553.0044 - _calculate_reconstruction_loss: 0.0093 - _calculate_kl_loss: 218.4396\n",
      "Epoch 21/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2944/3000 [============================>.] - ETA: 42s - loss: 9421.6885 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 218.3579 \n",
      "Epoch 00021: saving model to .\\weights\\weights-021-9418.66.h5\n",
      "\n",
      "Epoch 00021: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2285s 762ms/sample - loss: 9418.6554 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 218.3510\n",
      "Epoch 22/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9382.6228 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 216.7685 \n",
      "Epoch 00022: saving model to .\\weights\\weights-022-9379.70.h5\n",
      "\n",
      "Epoch 00022: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2280s 760ms/sample - loss: 9379.6979 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 216.7758\n",
      "Epoch 23/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9377.8481 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 216.4242 \n",
      "Epoch 00023: saving model to .\\weights\\weights-023-9373.69.h5\n",
      "\n",
      "Epoch 00023: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2279s 760ms/sample - loss: 9373.6906 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 216.4027\n",
      "Epoch 24/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9354.6148 - _calculate_reconstruction_loss: 0.0091 - _calculate_kl_loss: 215.8327 \n",
      "Epoch 00024: saving model to .\\weights\\weights-024-9345.33.h5\n",
      "\n",
      "Epoch 00024: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2284s 761ms/sample - loss: 9345.3299 - _calculate_reconstruction_loss: 0.0091 - _calculate_kl_loss: 215.7732\n",
      "Epoch 25/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9349.9913 - _calculate_reconstruction_loss: 0.0091 - _calculate_kl_loss: 214.6547 \n",
      "Epoch 00025: saving model to .\\weights\\weights-025-9383.09.h5\n",
      "\n",
      "Epoch 00025: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2283s 761ms/sample - loss: 9383.0851 - _calculate_reconstruction_loss: 0.0092 - _calculate_kl_loss: 214.6558\n",
      "Epoch 26/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9342.4749 - _calculate_reconstruction_loss: 0.0091 - _calculate_kl_loss: 212.7402 \n",
      "Epoch 00026: saving model to .\\weights\\weights-026-9346.56.h5\n",
      "\n",
      "Epoch 00026: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2287s 762ms/sample - loss: 9346.5624 - _calculate_reconstruction_loss: 0.0091 - _calculate_kl_loss: 212.7520\n",
      "Epoch 27/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9249.2393 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 214.1655 \n",
      "Epoch 00027: saving model to .\\weights\\weights-027-9249.89.h5\n",
      "\n",
      "Epoch 00027: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2279s 760ms/sample - loss: 9249.8896 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 214.1627\n",
      "Epoch 28/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9254.9896 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 212.5574 \n",
      "Epoch 00028: saving model to .\\weights\\weights-028-9242.47.h5\n",
      "\n",
      "Epoch 00028: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2291s 764ms/sample - loss: 9242.4681 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 212.4674\n",
      "Epoch 29/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9209.9887 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 212.9226 \n",
      "Epoch 00029: saving model to .\\weights\\weights-029-9199.33.h5\n",
      "\n",
      "Epoch 00029: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2278s 759ms/sample - loss: 9199.3306 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 212.8549\n",
      "Epoch 30/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9253.5469 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 211.5583 \n",
      "Epoch 00030: saving model to .\\weights\\weights-030-9243.81.h5\n",
      "\n",
      "Epoch 00030: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2287s 762ms/sample - loss: 9243.8149 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 211.5187\n",
      "Epoch 31/250\n",
      "2944/3000 [============================>.] - ETA: 42s - loss: 9150.8812 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 210.7791 \n",
      "Epoch 00031: saving model to .\\weights\\weights-031-9155.41.h5\n",
      "\n",
      "Epoch 00031: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2286s 762ms/sample - loss: 9155.4051 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 210.8319\n",
      "Epoch 32/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9191.2418 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.7650 \n",
      "Epoch 00032: saving model to .\\weights\\weights-032-9186.51.h5\n",
      "\n",
      "Epoch 00032: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2481s 827ms/sample - loss: 9186.5095 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.7052\n",
      "Epoch 33/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9164.4959 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 210.6529 \n",
      "Epoch 00033: saving model to .\\weights\\weights-033-9158.07.h5\n",
      "\n",
      "Epoch 00033: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2482s 827ms/sample - loss: 9158.0685 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 210.6349\n",
      "Epoch 34/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9212.0758 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.4403 \n",
      "Epoch 00034: saving model to .\\weights\\weights-034-9200.54.h5\n",
      "\n",
      "Epoch 00034: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2468s 823ms/sample - loss: 9200.5439 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.5002\n",
      "Epoch 35/250\n",
      "2944/3000 [============================>.] - ETA: 45s - loss: 9184.9997 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.1744 \n",
      "Epoch 00035: saving model to .\\weights\\weights-035-9191.48.h5\n",
      "\n",
      "Epoch 00035: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2461s 820ms/sample - loss: 9191.4779 - _calculate_reconstruction_loss: 0.0090 - _calculate_kl_loss: 209.1647\n",
      "Epoch 36/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9150.0693 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.1331 \n",
      "Epoch 00036: saving model to .\\weights\\weights-036-9145.37.h5\n",
      "\n",
      "Epoch 00036: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2474s 825ms/sample - loss: 9145.3653 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.1151\n",
      "Epoch 37/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9088.2362 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 206.7713 \n",
      "Epoch 00037: saving model to .\\weights\\weights-037-9087.75.h5\n",
      "\n",
      "Epoch 00037: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2514s 838ms/sample - loss: 9087.7489 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 206.7456\n",
      "Epoch 38/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9091.3299 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.4311 \n",
      "Epoch 00038: saving model to .\\weights\\weights-038-9081.81.h5\n",
      "\n",
      "Epoch 00038: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2508s 836ms/sample - loss: 9081.8133 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.4162\n",
      "Epoch 39/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9059.6510 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 206.2507 \n",
      "Epoch 00039: saving model to .\\weights\\weights-039-9061.88.h5\n",
      "\n",
      "Epoch 00039: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2500s 833ms/sample - loss: 9061.8776 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 206.3180\n",
      "Epoch 40/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2944/3000 [============================>.] - ETA: 45s - loss: 9083.9294 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.1001 \n",
      "Epoch 00040: saving model to .\\weights\\weights-040-9083.53.h5\n",
      "\n",
      "Epoch 00040: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2460s 820ms/sample - loss: 9083.5299 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 207.1259\n",
      "Epoch 41/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9089.8841 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 204.8193 \n",
      "Epoch 00041: saving model to .\\weights\\weights-041-9098.85.h5\n",
      "\n",
      "Epoch 00041: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2486s 829ms/sample - loss: 9098.8504 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 204.8281\n",
      "Epoch 42/250\n",
      "2944/3000 [============================>.] - ETA: 46s - loss: 9064.6432 - _calculate_reconstruction_loss: 0.0089 - _calculate_kl_loss: 204.8052 \n",
      "Epoch 00042: saving model to .\\weights\\weights-042-9051.51.h5\n",
      "\n",
      "Epoch 00042: saving model to .\\weights\\weights.h5\n",
      "3000/3000 [==============================] - 2498s 833ms/sample - loss: 9051.5055 - _calculate_reconstruction_loss: 0.0088 - _calculate_kl_loss: 204.7823\n",
      "Epoch 43/250\n",
      "1344/3000 [============>.................] - ETA: 24:24 - loss: 9049.9791 - _calculate_reconstruction_loss: 0.0088 - _calculate_kl_loss: 204.5126"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\" :\n",
    "  x_train = load_fsdd(SPECTROGRAMS_PATH)\n",
    "  autoencoder = train(x_train, LR, BS, EPOCHS, 0 )\n",
    "  autoencoder.save(\"sound_gen\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}