{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d422208b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import autoreload\n",
    "%load_ext autoreload\n",
    "# %autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2785de6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "934aac57",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c677106c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Input , Dense, Lambda, Concatenate, BatchNormalization\n",
    "from tensorflow.keras.activations import relu, sigmoid, tanh\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.optimizers import Adam, SGD, schedules\n",
    "from tensorflow_addons.optimizers import AdamW\n",
    "\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, TerminateOnNaN, TensorBoard, \\\n",
    "                                        EarlyStopping, ReduceLROnPlateau, CSVLogger, Callback\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "\n",
    "import gc\n",
    "\n",
    "from tqdm import tqdm "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1742e622",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a3141967",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from modules.ipynb\n",
      "importing Jupyter notebook from loss.ipynb\n",
      "importing Jupyter notebook from bandERB.ipynb\n",
      "importing Jupyter notebook from params.ipynb\n",
      "importing Jupyter notebook from utils.ipynb\n",
      "importing Jupyter notebook from dataloader.ipynb\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "\n",
    "from modules import convkxf, Conv2DBNAct, GroupGRULayer, GroupFC\n",
    "from loss import MaskLoss, LocalSnrTarget, DfAlphaLoss, SpectralLoss, SISDRloss, SpectralLoss_weight\n",
    "from utils import mask_operations, df_operations, synthesis_frame\n",
    "from params import model_params\n",
    "from dataloader import read_tfrecod_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5ff6bb12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from Callbacks.ipynb\n",
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "from Callbacks import WarmUpCosineDecayScheduler, BatchCallback, ClearMemory, ModelCheckpoint, TrainCallback, cosine_decay_with_warmup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f0e15be",
   "metadata": {},
   "source": [
    "# Read config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5bce3a7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = model_params('config.ini')\n",
    "initial_epoch = 39"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "806e9a85",
   "metadata": {},
   "source": [
    "# Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "818436e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = '/home2/user/myhsueh/dataset/tfrecord/v3/'\n",
    "\n",
    "training_set_path = [\n",
    "    dataset_path + 'boost/training',\n",
    "    dataset_path + 'random/training',\n",
    "    dataset_path + 'training/random',\n",
    "    dataset_path + 'training/car',\n",
    "    dataset_path + 'training/typing',\n",
    "    dataset_path + 'training/ambience'\n",
    "    \n",
    "]\n",
    "validation_set_path = [\n",
    "    dataset_path + 'boost/validation',\n",
    "    dataset_path + 'random/validation',\n",
    "    dataset_path + 'validation/random',\n",
    "    dataset_path + 'validation/car',\n",
    "    dataset_path + 'validation/typing',\n",
    "    dataset_path + 'validation/ambience'\n",
    "]\n",
    "combine_train, combine_val = False, False\n",
    "if len(training_set_path) > 1 : combine_train = True\n",
    "if len(validation_set_path) > 1 : combine_val = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e7ccd2b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "160\n",
      "TD_427-3-of-4.tfrecord, TD_444-3-of-4.tfrecord, TD_427-2-of-4.tfrecord, TD_433-3-of-4.tfrecord\n",
      "TD_431-4-of-4.tfrecord, TD_421-3-of-4.tfrecord, TD_425-4-of-4.tfrecord, TD_430-2-of-4.tfrecord\n",
      "TD_418-4-of-4.tfrecord, TD_422-1-of-4.tfrecord, TD_418-1-of-4.tfrecord, TD_448-3-of-4.tfrecord\n",
      "TD_420-4-of-4.tfrecord, TD_406-4-of-4.tfrecord, TD_426-2-of-4.tfrecord, TD_404-2-of-4.tfrecord\n",
      "TD_418-3-of-4.tfrecord, TD_415-3-of-4.tfrecord, TD_434-2-of-4.tfrecord, TD_428-3-of-4.tfrecord\n",
      "TD_401-4-of-4.tfrecord, TD_405-2-of-4.tfrecord, TD_405-3-of-4.tfrecord, TD_437-3-of-4.tfrecord\n",
      "TD_425-3-of-4.tfrecord, TD_400-3-of-4.tfrecord, TD_408-2-of-4.tfrecord, TD_430-3-of-4.tfrecord\n",
      "TD_438-1-of-4.tfrecord, TD_420-1-of-4.tfrecord, TD_434-3-of-4.tfrecord, TD_449-4-of-4.tfrecord\n",
      "TD_412-4-of-4.tfrecord, TD_441-1-of-4.tfrecord, TD_435-2-of-4.tfrecord, TD_429-4-of-4.tfrecord\n",
      "TD_448-1-of-4.tfrecord, TD_401-2-of-4.tfrecord, TD_403-2-of-4.tfrecord, TD_415-2-of-4.tfrecord\n",
      "TD_436-1-of-4.tfrecord, TD_445-3-of-4.tfrecord, TD_414-1-of-4.tfrecord, TD_434-4-of-4.tfrecord\n",
      "TD_416-2-of-4.tfrecord, TD_425-2-of-4.tfrecord, TD_401-3-of-4.tfrecord, TD_419-1-of-4.tfrecord\n",
      "TD_428-1-of-4.tfrecord, TD_432-1-of-4.tfrecord, TD_416-1-of-4.tfrecord, TD_432-4-of-4.tfrecord\n",
      "TD_424-3-of-4.tfrecord, TD_408-3-of-4.tfrecord, TD_410-4-of-4.tfrecord, TD_410-3-of-4.tfrecord\n",
      "TD_432-3-of-4.tfrecord, TD_407-3-of-4.tfrecord, TD_401-1-of-4.tfrecord, TD_435-3-of-4.tfrecord\n",
      "TD_431-2-of-4.tfrecord, TD_446-1-of-4.tfrecord, TD_419-4-of-4.tfrecord, TD_442-1-of-4.tfrecord\n",
      "TD_402-2-of-4.tfrecord, TD_415-4-of-4.tfrecord, TD_430-1-of-4.tfrecord, TD_420-3-of-4.tfrecord\n",
      "TD_400-2-of-4.tfrecord, TD_404-3-of-4.tfrecord, TD_448-4-of-4.tfrecord, TD_426-4-of-4.tfrecord\n",
      "TD_415-1-of-4.tfrecord, TD_422-2-of-4.tfrecord, TD_429-2-of-4.tfrecord, TD_426-1-of-4.tfrecord\n",
      "TD_417-1-of-4.tfrecord, TD_434-1-of-4.tfrecord, TD_400-4-of-4.tfrecord, TD_432-2-of-4.tfrecord\n",
      "TD_439-4-of-4.tfrecord, TD_406-2-of-4.tfrecord, TD_412-3-of-4.tfrecord, TD_400-1-of-4.tfrecord\n",
      "TD_410-1-of-4.tfrecord, TD_446-4-of-4.tfrecord, TD_419-2-of-4.tfrecord, TD_430-4-of-4.tfrecord\n",
      "TD_441-3-of-4.tfrecord, TD_422-3-of-4.tfrecord, TD_409-2-of-4.tfrecord, TD_445-1-of-4.tfrecord\n",
      "TD_421-2-of-4.tfrecord, TD_428-4-of-4.tfrecord, TD_409-4-of-4.tfrecord, TD_427-1-of-4.tfrecord\n",
      "TD_447-3-of-4.tfrecord, TD_413-4-of-4.tfrecord, TD_413-1-of-4.tfrecord, TD_443-4-of-4.tfrecord\n",
      "TD_408-4-of-4.tfrecord, TD_444-4-of-4.tfrecord, TD_426-3-of-4.tfrecord, TD_402-1-of-4.tfrecord\n",
      "TD_410-2-of-4.tfrecord, TD_403-1-of-4.tfrecord, TD_424-4-of-4.tfrecord, TD_424-2-of-4.tfrecord\n",
      "TD_440-1-of-4.tfrecord, TD_433-2-of-4.tfrecord, TD_429-3-of-4.tfrecord, TD_423-3-of-4.tfrecord\n",
      "TD_411-4-of-4.tfrecord, TD_429-1-of-4.tfrecord, TD_435-1-of-4.tfrecord, TD_437-2-of-4.tfrecord\n",
      "TD_412-1-of-4.tfrecord, TD_428-2-of-4.tfrecord, TD_414-4-of-4.tfrecord, TD_446-3-of-4.tfrecord\n",
      "TD_411-3-of-4.tfrecord, TD_439-3-of-4.tfrecord, TD_440-3-of-4.tfrecord, TD_417-2-of-4.tfrecord\n",
      "TD_447-4-of-4.tfrecord, TD_440-4-of-4.tfrecord, TD_441-4-of-4.tfrecord, TD_420-2-of-4.tfrecord\n",
      "TD_417-3-of-4.tfrecord, TD_443-2-of-4.tfrecord, TD_444-2-of-4.tfrecord, TD_437-4-of-4.tfrecord\n",
      "TD_439-2-of-4.tfrecord, TD_412-2-of-4.tfrecord, TD_413-3-of-4.tfrecord, TD_419-3-of-4.tfrecord\n",
      "TD_447-2-of-4.tfrecord, TD_417-4-of-4.tfrecord, TD_436-2-of-4.tfrecord, TD_402-3-of-4.tfrecord\n",
      "TD_414-2-of-4.tfrecord, TD_409-3-of-4.tfrecord, TD_409-1-of-4.tfrecord, TD_436-4-of-4.tfrecord\n",
      "TD_416-3-of-4.tfrecord, TD_407-2-of-4.tfrecord, TD_423-1-of-4.tfrecord, TD_443-3-of-4.tfrecord\n",
      "TD_433-1-of-4.tfrecord, TD_411-1-of-4.tfrecord, TD_440-2-of-4.tfrecord, TD_425-1-of-4.tfrecord\n",
      "TD_405-1-of-4.tfrecord, TD_435-4-of-4.tfrecord, TD_442-3-of-4.tfrecord, TD_404-1-of-4.tfrecord\n",
      "TD_442-4-of-4.tfrecord, TD_413-2-of-4.tfrecord, TD_446-2-of-4.tfrecord, TD_433-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "36\n",
      "TD_55-4-of-4.tfrecord, TD_52-2-of-4.tfrecord, TD_60-4-of-4.tfrecord, TD_59-2-of-4.tfrecord\n",
      "TD_60-1-of-4.tfrecord, TD_55-2-of-4.tfrecord, TD_50-3-of-4.tfrecord, TD_58-4-of-4.tfrecord\n",
      "TD_54-1-of-4.tfrecord, TD_51-2-of-4.tfrecord, TD_58-1-of-4.tfrecord, TD_52-4-of-4.tfrecord\n",
      "TD_59-4-of-4.tfrecord, TD_62-2-of-4.tfrecord, TD_56-4-of-4.tfrecord, TD_58-3-of-4.tfrecord\n",
      "TD_55-1-of-4.tfrecord, TD_60-2-of-4.tfrecord, TD_56-2-of-4.tfrecord, TD_51-3-of-4.tfrecord\n",
      "TD_51-1-of-4.tfrecord, TD_60-3-of-4.tfrecord, TD_59-1-of-4.tfrecord, TD_53-2-of-4.tfrecord\n",
      "TD_50-2-of-4.tfrecord, TD_59-3-of-4.tfrecord, TD_52-3-of-4.tfrecord, TD_62-4-of-4.tfrecord\n",
      "TD_55-3-of-4.tfrecord, TD_54-2-of-4.tfrecord, TD_53-1-of-4.tfrecord, TD_62-3-of-4.tfrecord\n",
      "TD_54-4-of-4.tfrecord, TD_58-2-of-4.tfrecord, TD_62-1-of-4.tfrecord, TD_53-3-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "168\n",
      "TD_41-2-of-4.tfrecord, TD_40-2-of-4.tfrecord, TD_34-2-of-4.tfrecord, TD_1-2-of-4.tfrecord\n",
      "TD_40-1-of-4.tfrecord, TD_34-4-of-4.tfrecord, TD_34-1-of-4.tfrecord, TD_32-3-of-4.tfrecord\n",
      "TD_15-4-of-4.tfrecord, TD_4-3-of-4.tfrecord, TD_49-4-of-4.tfrecord, TD_31-4-of-4.tfrecord\n",
      "TD_5-1-of-4.tfrecord, TD_14-2-of-4.tfrecord, TD_12-2-of-4.tfrecord, TD_20-2-of-4.tfrecord\n",
      "TD_19-1-of-4.tfrecord, TD_24-4-of-4.tfrecord, TD_0-3-of-4.tfrecord, TD_43-1-of-4.tfrecord\n",
      "TD_41-3-of-4.tfrecord, TD_26-4-of-4.tfrecord, TD_28-2-of-4.tfrecord, TD_2-2-of-4.tfrecord\n",
      "TD_1-3-of-4.tfrecord, TD_7-4-of-4.tfrecord, TD_35-4-of-4.tfrecord, TD_42-3-of-4.tfrecord\n",
      "TD_43-4-of-4.tfrecord, TD_26-3-of-4.tfrecord, TD_3-2-of-4.tfrecord, TD_21-4-of-4.tfrecord\n",
      "TD_33-3-of-4.tfrecord, TD_20-1-of-4.tfrecord, TD_35-2-of-4.tfrecord, TD_18-1-of-4.tfrecord\n",
      "TD_32-2-of-4.tfrecord, TD_42-1-of-4.tfrecord, TD_49-2-of-4.tfrecord, TD_17-3-of-4.tfrecord\n",
      "TD_8-2-of-4.tfrecord, TD_35-3-of-4.tfrecord, TD_14-1-of-4.tfrecord, TD_22-4-of-4.tfrecord\n",
      "TD_6-3-of-4.tfrecord, TD_16-4-of-4.tfrecord, TD_25-2-of-4.tfrecord, TD_25-4-of-4.tfrecord\n",
      "TD_46-4-of-4.tfrecord, TD_17-2-of-4.tfrecord, TD_19-2-of-4.tfrecord, TD_11-3-of-4.tfrecord\n",
      "TD_26-1-of-4.tfrecord, TD_47-2-of-4.tfrecord, TD_16-1-of-4.tfrecord, TD_18-3-of-4.tfrecord\n",
      "TD_10-2-of-4.tfrecord, TD_3-1-of-4.tfrecord, TD_16-2-of-4.tfrecord, TD_11-1-of-4.tfrecord\n",
      "TD_29-1-of-4.tfrecord, TD_13-3-of-4.tfrecord, TD_11-2-of-4.tfrecord, TD_42-2-of-4.tfrecord\n",
      "TD_47-3-of-4.tfrecord, TD_44-2-of-4.tfrecord, TD_49-1-of-4.tfrecord, TD_48-2-of-4.tfrecord\n",
      "TD_3-3-of-4.tfrecord, TD_34-3-of-4.tfrecord, TD_4-4-of-4.tfrecord, TD_40-3-of-4.tfrecord\n",
      "TD_28-3-of-4.tfrecord, TD_40-4-of-4.tfrecord, TD_5-4-of-4.tfrecord, TD_1-4-of-4.tfrecord\n",
      "TD_31-1-of-4.tfrecord, TD_46-1-of-4.tfrecord, TD_29-4-of-4.tfrecord, TD_45-4-of-4.tfrecord\n",
      "TD_37-4-of-4.tfrecord, TD_41-4-of-4.tfrecord, TD_45-2-of-4.tfrecord, TD_44-1-of-4.tfrecord\n",
      "TD_42-4-of-4.tfrecord, TD_9-3-of-4.tfrecord, TD_32-4-of-4.tfrecord, TD_43-3-of-4.tfrecord\n",
      "TD_6-2-of-4.tfrecord, TD_28-4-of-4.tfrecord, TD_7-3-of-4.tfrecord, TD_14-4-of-4.tfrecord\n",
      "TD_16-3-of-4.tfrecord, TD_15-3-of-4.tfrecord, TD_3-4-of-4.tfrecord, TD_7-1-of-4.tfrecord\n",
      "TD_38-2-of-4.tfrecord, TD_6-4-of-4.tfrecord, TD_2-3-of-4.tfrecord, TD_4-2-of-4.tfrecord\n",
      "TD_21-2-of-4.tfrecord, TD_33-2-of-4.tfrecord, TD_37-1-of-4.tfrecord, TD_18-2-of-4.tfrecord\n",
      "TD_38-3-of-4.tfrecord, TD_21-1-of-4.tfrecord, TD_30-4-of-4.tfrecord, TD_14-3-of-4.tfrecord\n",
      "TD_17-1-of-4.tfrecord, TD_48-3-of-4.tfrecord, TD_31-2-of-4.tfrecord, TD_10-3-of-4.tfrecord\n",
      "TD_39-4-of-4.tfrecord, TD_13-1-of-4.tfrecord, TD_13-4-of-4.tfrecord, TD_33-4-of-4.tfrecord\n",
      "TD_2-1-of-4.tfrecord, TD_44-4-of-4.tfrecord, TD_15-2-of-4.tfrecord, TD_11-4-of-4.tfrecord\n",
      "TD_24-3-of-4.tfrecord, TD_9-2-of-4.tfrecord, TD_27-1-of-4.tfrecord, TD_6-1-of-4.tfrecord\n",
      "TD_26-2-of-4.tfrecord, TD_43-2-of-4.tfrecord, TD_33-1-of-4.tfrecord, TD_21-3-of-4.tfrecord\n",
      "TD_22-3-of-4.tfrecord, TD_36-1-of-4.tfrecord, TD_45-1-of-4.tfrecord, TD_23-2-of-4.tfrecord\n",
      "TD_2-4-of-4.tfrecord, TD_30-1-of-4.tfrecord, TD_44-3-of-4.tfrecord, TD_12-3-of-4.tfrecord\n",
      "TD_22-1-of-4.tfrecord, TD_27-4-of-4.tfrecord, TD_5-2-of-4.tfrecord, TD_23-4-of-4.tfrecord\n",
      "TD_4-1-of-4.tfrecord, TD_29-2-of-4.tfrecord, TD_47-1-of-4.tfrecord, TD_10-4-of-4.tfrecord\n",
      "TD_18-4-of-4.tfrecord, TD_25-3-of-4.tfrecord, TD_39-2-of-4.tfrecord, TD_46-3-of-4.tfrecord\n",
      "TD_30-3-of-4.tfrecord, TD_12-1-of-4.tfrecord, TD_28-1-of-4.tfrecord, TD_41-1-of-4.tfrecord\n",
      "TD_9-1-of-4.tfrecord, TD_39-3-of-4.tfrecord, TD_31-3-of-4.tfrecord, TD_37-3-of-4.tfrecord\n",
      "TD_0-2-of-4.tfrecord, TD_38-4-of-4.tfrecord, TD_24-1-of-4.tfrecord, TD_22-2-of-4.tfrecord\n",
      "TD_27-2-of-4.tfrecord, TD_32-1-of-4.tfrecord, TD_48-4-of-4.tfrecord, TD_48-1-of-4.tfrecord\n",
      "TD_8-3-of-4.tfrecord, TD_35-1-of-4.tfrecord, TD_5-3-of-4.tfrecord, TD_27-3-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "32\n",
      "TD_55-4-of-4.tfrecord, TD_52-2-of-4.tfrecord, TD_57-3-of-4.tfrecord, TD_50-3-of-4.tfrecord\n",
      "TD_58-4-of-4.tfrecord, TD_51-2-of-4.tfrecord, TD_57-2-of-4.tfrecord, TD_58-1-of-4.tfrecord\n",
      "TD_59-4-of-4.tfrecord, TD_56-1-of-4.tfrecord, TD_58-3-of-4.tfrecord, TD_56-3-of-4.tfrecord\n",
      "TD_55-1-of-4.tfrecord, TD_56-2-of-4.tfrecord, TD_51-3-of-4.tfrecord, TD_51-1-of-4.tfrecord\n",
      "TD_59-1-of-4.tfrecord, TD_53-2-of-4.tfrecord, TD_50-2-of-4.tfrecord, TD_57-1-of-4.tfrecord\n",
      "TD_59-3-of-4.tfrecord, TD_52-3-of-4.tfrecord, TD_55-3-of-4.tfrecord, TD_54-3-of-4.tfrecord\n",
      "TD_50-1-of-4.tfrecord, TD_54-2-of-4.tfrecord, TD_53-1-of-4.tfrecord, TD_54-4-of-4.tfrecord\n",
      "TD_58-2-of-4.tfrecord, TD_53-4-of-4.tfrecord, TD_53-3-of-4.tfrecord, TD_51-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "32\n",
      "TD_66-3-of-4.tfrecord, TD_60-4-of-4.tfrecord, TD_69-1-of-4.tfrecord, TD_60-1-of-4.tfrecord\n",
      "TD_61-2-of-4.tfrecord, TD_65-4-of-4.tfrecord, TD_63-3-of-4.tfrecord, TD_67-4-of-4.tfrecord\n",
      "TD_65-2-of-4.tfrecord, TD_64-3-of-4.tfrecord, TD_68-3-of-4.tfrecord, TD_61-4-of-4.tfrecord\n",
      "TD_68-4-of-4.tfrecord, TD_61-1-of-4.tfrecord, TD_67-3-of-4.tfrecord, TD_62-2-of-4.tfrecord\n",
      "TD_66-4-of-4.tfrecord, TD_67-2-of-4.tfrecord, TD_60-2-of-4.tfrecord, TD_66-2-of-4.tfrecord\n",
      "TD_63-4-of-4.tfrecord, TD_64-4-of-4.tfrecord, TD_64-1-of-4.tfrecord, TD_68-1-of-4.tfrecord\n",
      "TD_69-3-of-4.tfrecord, TD_63-2-of-4.tfrecord, TD_69-4-of-4.tfrecord, TD_62-4-of-4.tfrecord\n",
      "TD_64-2-of-4.tfrecord, TD_65-3-of-4.tfrecord, TD_63-1-of-4.tfrecord, TD_62-1-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/training', '/home2/user/myhsueh/dataset/tfrecord/v3/random/training', '/home2/user/myhsueh/dataset/tfrecord/v3/training/random', '/home2/user/myhsueh/dataset/tfrecord/v3/training/car', '/home2/user/myhsueh/dataset/tfrecord/v3/training/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/training/ambience'] , file directory exist: True\n",
      "58\n",
      "TD_283-1-of-4.tfrecord, TD_280-2-of-4.tfrecord, TD_282-2-of-4.tfrecord, TD_279-1-of-4.tfrecord\n",
      "TD_281-1-of-4.tfrecord, TD_280-1-of-4.tfrecord, TD_280-4-of-4.tfrecord, TD_273-3-of-4.tfrecord\n",
      "TD_273-2-of-4.tfrecord, TD_287-2-of-4.tfrecord, TD_271-3-of-4.tfrecord, TD_275-4-of-4.tfrecord\n",
      "TD_279-4-of-4.tfrecord, TD_277-2-of-4.tfrecord, TD_290-4-of-4.tfrecord, TD_287-1-of-4.tfrecord\n",
      "TD_271-4-of-4.tfrecord, TD_275-1-of-4.tfrecord, TD_284-1-of-4.tfrecord, TD_272-2-of-4.tfrecord\n",
      "TD_279-3-of-4.tfrecord, TD_287-4-of-4.tfrecord, TD_281-2-of-4.tfrecord, TD_271-2-of-4.tfrecord\n",
      "TD_274-4-of-4.tfrecord, TD_273-1-of-4.tfrecord, TD_270-1-of-4.tfrecord, TD_276-4-of-4.tfrecord\n",
      "TD_286-4-of-4.tfrecord, TD_284-3-of-4.tfrecord, TD_278-3-of-4.tfrecord, TD_278-4-of-4.tfrecord\n",
      "TD_282-3-of-4.tfrecord, TD_290-3-of-4.tfrecord, TD_284-2-of-4.tfrecord, TD_286-2-of-4.tfrecord\n",
      "TD_270-2-of-4.tfrecord, TD_281-4-of-4.tfrecord, TD_287-3-of-4.tfrecord, TD_282-1-of-4.tfrecord\n",
      "TD_277-4-of-4.tfrecord, TD_277-3-of-4.tfrecord, TD_272-1-of-4.tfrecord, TD_276-2-of-4.tfrecord\n",
      "TD_280-3-of-4.tfrecord, TD_274-3-of-4.tfrecord, TD_270-4-of-4.tfrecord, TD_283-3-of-4.tfrecord\n",
      "TD_284-4-of-4.tfrecord, TD_275-2-of-4.tfrecord, TD_281-3-of-4.tfrecord, TD_283-4-of-4.tfrecord\n",
      "TD_276-1-of-4.tfrecord, TD_286-3-of-4.tfrecord, TD_271-1-of-4.tfrecord, TD_272-4-of-4.tfrecord\n",
      "TD_273-4-of-4.tfrecord, TD_274-1-of-4.tfrecord, Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "dataset = read_tfrecod_data(training_set_path, training=True, repeat=p.epochs-initial_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0efbb3db",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "40\n",
      "TD_421-1-of-4.tfrecord, TD_423-2-of-4.tfrecord, TD_438-3-of-4.tfrecord, TD_406-3-of-4.tfrecord\n",
      "TD_403-4-of-4.tfrecord, TD_445-4-of-4.tfrecord, TD_447-1-of-4.tfrecord, TD_404-4-of-4.tfrecord\n",
      "TD_424-1-of-4.tfrecord, TD_437-1-of-4.tfrecord, TD_449-1-of-4.tfrecord, TD_436-3-of-4.tfrecord\n",
      "TD_405-4-of-4.tfrecord, TD_406-1-of-4.tfrecord, TD_431-1-of-4.tfrecord, TD_408-1-of-4.tfrecord\n",
      "TD_403-3-of-4.tfrecord, TD_411-2-of-4.tfrecord, TD_441-2-of-4.tfrecord, TD_438-4-of-4.tfrecord\n",
      "TD_442-2-of-4.tfrecord, TD_421-4-of-4.tfrecord, TD_418-2-of-4.tfrecord, TD_448-2-of-4.tfrecord\n",
      "TD_414-3-of-4.tfrecord, TD_431-3-of-4.tfrecord, TD_427-4-of-4.tfrecord, TD_407-1-of-4.tfrecord\n",
      "TD_416-4-of-4.tfrecord, TD_449-2-of-4.tfrecord, TD_402-4-of-4.tfrecord, TD_423-4-of-4.tfrecord\n",
      "TD_438-2-of-4.tfrecord, TD_445-2-of-4.tfrecord, TD_443-1-of-4.tfrecord, TD_422-4-of-4.tfrecord\n",
      "TD_439-1-of-4.tfrecord, TD_449-3-of-4.tfrecord, TD_444-1-of-4.tfrecord, TD_407-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "8\n",
      "TD_52-1-of-4.tfrecord, TD_56-1-of-4.tfrecord, TD_50-4-of-4.tfrecord, TD_56-3-of-4.tfrecord\n",
      "TD_54-3-of-4.tfrecord, TD_50-1-of-4.tfrecord, TD_53-4-of-4.tfrecord, TD_51-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "40\n",
      "TD_30-2-of-4.tfrecord, TD_10-1-of-4.tfrecord, TD_20-4-of-4.tfrecord, TD_37-2-of-4.tfrecord\n",
      "TD_35-4-of-4.tfrecord, TD_24-2-of-4.tfrecord, TD_47-4-of-4.tfrecord, TD_8-4-of-4.tfrecord\n",
      "TD_35-2-of-4.tfrecord, TD_45-3-of-4.tfrecord, TD_46-2-of-4.tfrecord, TD_29-3-of-4.tfrecord\n",
      "TD_26-1-of-4.tfrecord, TD_19-4-of-4.tfrecord, TD_11-1-of-4.tfrecord, TD_42-2-of-4.tfrecord\n",
      "TD_38-1-of-4.tfrecord, TD_49-3-of-4.tfrecord, TD_41-4-of-4.tfrecord, TD_28-4-of-4.tfrecord\n",
      "TD_1-1-of-4.tfrecord, TD_17-4-of-4.tfrecord, TD_15-1-of-4.tfrecord, TD_13-2-of-4.tfrecord\n",
      "TD_23-1-of-4.tfrecord, TD_0-1-of-4.tfrecord, TD_36-3-of-4.tfrecord, TD_23-3-of-4.tfrecord\n",
      "TD_20-3-of-4.tfrecord, TD_7-2-of-4.tfrecord, TD_25-1-of-4.tfrecord, TD_19-3-of-4.tfrecord\n",
      "TD_0-4-of-4.tfrecord, TD_36-2-of-4.tfrecord, TD_9-4-of-4.tfrecord, TD_8-1-of-4.tfrecord\n",
      "TD_39-1-of-4.tfrecord, TD_27-2-of-4.tfrecord, TD_12-4-of-4.tfrecord, TD_36-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "8\n",
      "TD_57-4-of-4.tfrecord, TD_52-1-of-4.tfrecord, TD_59-2-of-4.tfrecord, TD_55-2-of-4.tfrecord\n",
      "TD_54-1-of-4.tfrecord, TD_52-4-of-4.tfrecord, TD_56-4-of-4.tfrecord, TD_50-4-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "8\n",
      "TD_65-1-of-4.tfrecord, TD_69-2-of-4.tfrecord, TD_61-3-of-4.tfrecord, TD_60-3-of-4.tfrecord\n",
      "TD_66-1-of-4.tfrecord, TD_67-1-of-4.tfrecord, TD_68-2-of-4.tfrecord, TD_62-3-of-4.tfrecord\n",
      "Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n",
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n",
      "['/home2/user/myhsueh/dataset/tfrecord/v3/boost/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/random/validation', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/random', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/car', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/typing', '/home2/user/myhsueh/dataset/tfrecord/v3/validation/ambience'] , file directory exist: True\n",
      "14\n",
      "TD_270-3-of-4.tfrecord, TD_278-2-of-4.tfrecord, TD_274-2-of-4.tfrecord, TD_278-1-of-4.tfrecord\n",
      "TD_282-4-of-4.tfrecord, TD_290-2-of-4.tfrecord, TD_286-1-of-4.tfrecord, TD_290-1-of-4.tfrecord\n",
      "TD_275-3-of-4.tfrecord, TD_279-2-of-4.tfrecord, TD_283-2-of-4.tfrecord, TD_272-3-of-4.tfrecord\n",
      "TD_276-3-of-4.tfrecord, TD_277-1-of-4.tfrecord, Tensor(\"Reshape:0\", shape=(1, 1024), dtype=float32)\n",
      "signal:  Tensor(\"ParseSingleExample/ParseExample/ParseExampleV2:0\", shape=(144000,), dtype=float32)\n",
      "fft:  Tensor(\"mul_1:0\", shape=(282, 513), dtype=complex64)\n",
      "Amp:  Tensor(\"add:0\", shape=(282, 513), dtype=float32)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done erb\n",
      "done spec\n",
      "Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "ERB shape: (282, 32, 1) Tensor(\"Squeeze_282:0\", shape=(282, 32, 1), dtype=float32)\n",
      "NOISY_SPEC_df shape: (282, 96, 2) Tensor(\"Reshape_4:0\", shape=(282, 96, 2), dtype=float32)\n",
      "CLEAN_SPEC shape: (282, 513, 2) Tensor(\"Reshape_6:0\", shape=(282, 513, 2), dtype=float32)\n",
      "NOISY_SPEC shape: (282, 513, 2) Tensor(\"Reshape_5:0\", shape=(282, 513, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "val_dataset = read_tfrecod_data(validation_set_path, training=False, repeat=p.epochs-initial_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "86dff8c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training step: 7609 , validation step: 1843\n"
     ]
    }
   ],
   "source": [
    "each_tfr_batch = 6000 // 4 // p.length_sec\n",
    "train_count, val_count = 0, 0\n",
    "if combine_train:\n",
    "    for i in training_set_path:\n",
    "        train_count += len(os.listdir(i))\n",
    "else:\n",
    "    train_count = len(os.listdir(training_set_path[0]))\n",
    "    \n",
    "if combine_val:\n",
    "    for i in validation_set_path:\n",
    "        val_count += len(os.listdir(i))\n",
    "else:\n",
    "    val_count = len(os.listdir(validation_set_path[0]))\n",
    "\n",
    "steps_per_epoch = train_count*each_tfr_batch//p.batch_size \n",
    "validation_steps = val_count*each_tfr_batch//p.batch_size \n",
    "print('training step: {} , validation step: {}'.format(steps_per_epoch,validation_steps))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd863e94",
   "metadata": {},
   "source": [
    "# Model construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9bcef9f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "BN_type = \"normal\" # range normal\n",
    "p.fc_group = 8\n",
    "p.gru_group = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ed6f8283",
   "metadata": {},
   "outputs": [],
   "source": [
    "erb_inputs = Input(shape=(None, p.nb_erb, 1), name='ERB_input')\n",
    "spec_inputs = Input(shape=(None, p.nb_df, 2), name='spec_input') # complex\n",
    "clean_spec = Input(shape=(None, p.fft_size//2+1, 2), name='clean_spec') # complex\n",
    "noisy_spec = Input(shape=(None, p.fft_size//2+1, 2), name='noisy_spec') # complex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "88464f21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'filters': 64, 'groups': 1, 'kernel_size': (3, 3), 'strides': (1, 1), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'conv0_encoder'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 3), 'strides': (1, 2), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'conv1_encoder'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 3), 'strides': (1, 2), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'conv2_encoder'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 3), 'strides': (1, 1), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'conv3_encoder'}\n",
      "{'filters': 64, 'groups': 2, 'kernel_size': (3, 3), 'strides': (1, 1), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'df_conv0_encoder'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 3), 'strides': (1, 2), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'df_conv1_encoder'}\n"
     ]
    }
   ],
   "source": [
    "erb_conv0 = convkxf(erb_inputs, p.conv_out_ch, k=3, f=3, fstride=1, bias=False, batch_norm=True, training=True, \n",
    "                     BN_type = BN_type, name='conv0_encoder')\n",
    "erb_conv1 = convkxf(erb_conv0, p.conv_out_ch, k=1, f=3, fstride=2, bias=False, batch_norm=True,\n",
    "                     BN_type = BN_type, name='conv1_encoder')\n",
    "erb_conv2 = convkxf(erb_conv1, p.conv_out_ch, k=1, f=3, fstride=2, bias=False, batch_norm=True,\n",
    "                     BN_type = BN_type, name='conv2_encoder')\n",
    "erb_conv3 = convkxf(erb_conv2, p.conv_out_ch, k=1, f=3, fstride=1, bias=False, batch_norm=True,\n",
    "                     BN_type = BN_type, name='conv3_encoder')\n",
    "\n",
    "df_conv0 = convkxf(spec_inputs, p.conv_out_ch, k=3, f=3, fstride=1, batch_norm=True, training=True,\n",
    "                    BN_type = BN_type, name='df_conv0_encoder')\n",
    "df_conv1 = convkxf(df_conv0, p.conv_out_ch, k=1, f=3, fstride=2, batch_norm=True,\n",
    "                    BN_type = BN_type, name='df_conv1_encoder')\n",
    "\n",
    "shape = [tf.shape(df_conv1)[l] for l in range(4)]\n",
    "df_conv1 = tf.transpose(df_conv1,(0,1,3,2))\n",
    "df_conv1 = tf.reshape(df_conv1, [shape[0], shape[1], shape[2]*shape[3]])\n",
    "\n",
    "cemb = GroupFC(df_conv1, p.fc_hidden, p.fc_group, name='GFC_encoder')\n",
    "    \n",
    "shape = [tf.shape(erb_conv3)[l] for l in range(4)]\n",
    "emb = Concatenate()([tf.reshape(tf.transpose(erb_conv3,(0,1,3,2)), [shape[0], shape[1], shape[2]*shape[3]]), cemb])\n",
    "\n",
    "if p.model_ver == \"3.0\":\n",
    "    embfc = GroupFC(emb, p.gru_hidden, p.fc_group, name='GFC_emb')\n",
    "    GRU_emb = GroupGRULayer(embfc, p.gru_hidden, p.gru_group, num_layer=1, name='GGRU0', add_output=True)\n",
    "else:\n",
    "    GRU_emb = GroupGRULayer(emb, p.gru_hidden, p.gru_group, num_layer=1, name='GGRU0', add_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3d225d12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 1), 'strides': 1, 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convp3'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 3), 'strides': (1, 1), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convt3'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 1), 'strides': 1, 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convp2'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 1), 'strides': 1, 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convp1'}\n",
      "{'filters': 64, 'groups': 64, 'kernel_size': (1, 1), 'strides': 1, 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convp0'}\n",
      "{'filters': 1, 'groups': 1, 'kernel_size': (1, 3), 'strides': (1, 1), 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'mask_out'}\n"
     ]
    }
   ],
   "source": [
    "GRU_emb1 = GroupGRULayer(GRU_emb, p.gru_hidden, p.gru_group, num_layer=2, name='GGRU01', add_output=True)\n",
    "de_emb = GroupFC(GRU_emb1, p.fc_hidden, p.fc_group, activation='relu', \n",
    "                 name='GFC_decoder')\n",
    "\n",
    "shape = [tf.shape(erb_conv3)[l] for l in range(4)]\n",
    "emb_decoder = tf.reshape(de_emb, shape = [shape[0], shape[1], shape[2], -1], name='decoder_reshape')\n",
    "\n",
    "kwargs = {\n",
    "    \"k\": 1,\n",
    "    \"batch_norm\": True,\n",
    "    \"BN_type\": BN_type,\n",
    "}\n",
    "tkwargs = {\n",
    "    \"k\": 1,\n",
    "    \"batch_norm\": True,\n",
    "    \"mode\": \"transposed\",\n",
    "    \"BN_type\": BN_type,\n",
    "}\n",
    "pkwargs = {\n",
    "    \"k\": 1,\n",
    "    \"f\": 1,\n",
    "    \"batch_norm\": True,\n",
    "    \"BN_type\": BN_type,\n",
    "}\n",
    "\n",
    "convp3 = convkxf(erb_conv3, out_ch=p.conv_out_ch, name='convp3', **pkwargs)  # Conv\n",
    "vt3_in = emb_decoder + convp3\n",
    "convt3 = convkxf(vt3_in, out_ch=p.conv_out_ch, fstride=1, name='convt3', **kwargs) #ConvT\n",
    "\n",
    "convp2 = convkxf(erb_conv2, out_ch=p.conv_out_ch, name='convp2', **pkwargs) # Conv\n",
    "vt2_in = convt3 + convp2\n",
    "convt2 = convkxf(vt2_in, out_ch=p.conv_out_ch, name='convt2', **tkwargs)\n",
    "\n",
    "convp1 = convkxf(erb_conv1, out_ch=p.conv_out_ch, name='convp1', **pkwargs) # Conv\n",
    "vt1_in = convt2  + convp1\n",
    "convt1 = convkxf(vt1_in, out_ch=p.conv_out_ch, name='convt1', **tkwargs)\n",
    "\n",
    "convp0 = convkxf(erb_conv0, out_ch=p.conv_out_ch, name='convp0', **pkwargs) # Conv\n",
    "vt0_in = convt1 + convp0\n",
    "mask_out = convkxf(vt0_in, out_ch=1, k=1, fstride=1, act='sigmoid', batch_norm=False, \n",
    "                   reshape=True, name='mask_out')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c6b82912",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'filters': 10, 'groups': 2, 'kernel_size': (1, 1), 'strides': 1, 'use_bias': False, 'kernel_initializer': 'he_normal', 'padding': 'valid', 'name': 'convp_DfDecoder'}\n"
     ]
    }
   ],
   "source": [
    "GRU_emb2 = GroupGRULayer(GRU_emb, p.gru_hidden, p.gru_group, num_layer=2, name='GGRU1', add_output=True)\n",
    "convp = convkxf(df_conv0, 2*p.df_order, k=1, f=1, complex_in=True, batch_norm=True, \n",
    "                BN_type = BN_type, name='convp_DfDecoder') \n",
    "convp = tf.transpose(convp, (0,1,3,2))\n",
    "\n",
    "df_alpha = Dense(1, name='convp_alpha', activation='sigmoid')(GRU_emb2)\n",
    "c = GroupFC(GRU_emb2, p.nb_df*p.df_order*2, p.fc_group, activation='tanh', name='GFC_c')\n",
    "\n",
    "shape = [tf.shape(c)[k] for k in range(2)]\n",
    "c = tf.reshape(c,(shape[0], shape[1], p.df_order*2, p.nb_df))\n",
    "\n",
    "c = tf.reshape(c + convp,(shape[0], shape[1], p.df_order, 2, p.nb_df))\n",
    "\n",
    "df_coeff = tf.transpose(c, [0, 1, 2, 4, 3], name='output_c')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38180c11",
   "metadata": {},
   "source": [
    "# Enhance operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "11541142",
   "metadata": {},
   "outputs": [],
   "source": [
    "spec = mask_operations(noisy_spec, mask_out) # mask gain\n",
    "enhanced = df_operations(spec, df_coeff, df_alpha) # deep filter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec8c397d",
   "metadata": {},
   "source": [
    "# Callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e19617a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks_nan = TerminateOnNaN()\n",
    "callbacks_earlystop = EarlyStopping(patience=10, mode='min', restore_best_weights=True)\n",
    "# Memory clear\n",
    "Memory_callback=ClearMemory()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "037927ff",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2b7e425a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "maskloss = Lambda(lambda x:MaskLoss(*x, factor=p.mask_factor, r=p.mask_gamma), \n",
    "                  name='maskloss')([mask_out, clean_spec, noisy_spec])\n",
    "spectralloss = Lambda(lambda x:SpectralLoss(*x, gamma=p.df_gamma, factor_mag=p.df_factor, factor_img=p.df_factor), \n",
    "                      name='spectralloss')([enhanced, clean_spec])\n",
    "# spectralloss2 = Lambda(lambda x:SpectralLoss(*x, gamma=p.df_gamma*2, factor_mag=p.df_factor, factor_img=p.df_factor), \n",
    "#                       name='spectralloss2')([enhanced, clean_spec])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b2d2b67",
   "metadata": {},
   "source": [
    "# Model compile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "635ef4ec",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "inputs = [erb_inputs, spec_inputs, clean_spec, noisy_spec]\n",
    "outputs = [maskloss,spectralloss]\n",
    "model=Model(inputs=inputs, outputs=outputs, name='DfNet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "39435d21",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"DfNet\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " ERB_input (InputLayer)         [(None, None, 32, 1  0           []                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad (TFOpLambda)  (None, None, 32, 1)  0           ['ERB_input[0][0]']              \n",
      "                                                                                                  \n",
      " spec_input (InputLayer)        [(None, None, 96, 2  0           []                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_1 (TFOpLambda  (None, None, 34, 1)  0          ['tf.compat.v1.pad[0][0]']       \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_5 (TFOpLambda  (None, None, 96, 2)  0          ['spec_input[0][0]']             \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " conv0_encoder (Conv2D)         (None, None, 32, 64  576         ['tf.compat.v1.pad_1[0][0]']     \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_6 (TFOpLambda  (None, None, 98, 2)  0          ['tf.compat.v1.pad_5[0][0]']     \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " conv0_encoderBatchNorm (BatchN  (None, None, 32, 64  256        ['conv0_encoder[0][0]']          \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " df_conv0_encoder (Conv2D)      (None, None, 96, 64  576         ['tf.compat.v1.pad_6[0][0]']     \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu (TFOpLambda)        (None, None, 32, 64  0           ['conv0_encoderBatchNorm[0][0]'] \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " df_conv0_encoder_1x1 (Conv2D)  (None, None, 96, 64  4096        ['df_conv0_encoder[0][0]']       \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_2 (TFOpLambda  (None, None, 34, 64  0          ['tf.nn.relu[0][0]']             \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " df_conv0_encoderBatchNorm (Bat  (None, None, 96, 64  256        ['df_conv0_encoder_1x1[0][0]']   \n",
      " chNormalization)               )                                                                 \n",
      "                                                                                                  \n",
      " conv1_encoder (Conv2D)         (None, None, 16, 64  192         ['tf.compat.v1.pad_2[0][0]']     \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_4 (TFOpLambda)      (None, None, 96, 64  0           ['df_conv0_encoderBatchNorm[0][0]\n",
      "                                )                                ']                               \n",
      "                                                                                                  \n",
      " conv1_encoder_1x1 (Conv2D)     (None, None, 16, 64  4096        ['conv1_encoder[0][0]']          \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_7 (TFOpLambda  (None, None, 98, 64  0          ['tf.nn.relu_4[0][0]']           \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " conv1_encoderBatchNorm (BatchN  (None, None, 16, 64  256        ['conv1_encoder_1x1[0][0]']      \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " df_conv1_encoder (Conv2D)      (None, None, 48, 64  192         ['tf.compat.v1.pad_7[0][0]']     \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_1 (TFOpLambda)      (None, None, 16, 64  0           ['conv1_encoderBatchNorm[0][0]'] \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " df_conv1_encoder_1x1 (Conv2D)  (None, None, 48, 64  4096        ['df_conv1_encoder[0][0]']       \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_3 (TFOpLambda  (None, None, 18, 64  0          ['tf.nn.relu_1[0][0]']           \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " df_conv1_encoderBatchNorm (Bat  (None, None, 48, 64  256        ['df_conv1_encoder_1x1[0][0]']   \n",
      " chNormalization)               )                                                                 \n",
      "                                                                                                  \n",
      " conv2_encoder (Conv2D)         (None, None, 8, 64)  192         ['tf.compat.v1.pad_3[0][0]']     \n",
      "                                                                                                  \n",
      " tf.nn.relu_5 (TFOpLambda)      (None, None, 48, 64  0           ['df_conv1_encoderBatchNorm[0][0]\n",
      "                                )                                ']                               \n",
      "                                                                                                  \n",
      " conv2_encoder_1x1 (Conv2D)     (None, None, 8, 64)  4096        ['conv2_encoder[0][0]']          \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_2 (TFOpLamb  (4,)                0           ['tf.nn.relu_5[0][0]']           \n",
      " da)                                                                                              \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_3 (TFOpLamb  (4,)                0           ['tf.nn.relu_5[0][0]']           \n",
      " da)                                                                                              \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " conv2_encoderBatchNorm (BatchN  (None, None, 8, 64)  256        ['conv2_encoder_1x1[0][0]']      \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape (TFOpLambda  (4,)                0           ['tf.nn.relu_5[0][0]']           \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_1 (TFOpLamb  (4,)                0           ['tf.nn.relu_5[0][0]']           \n",
      " da)                                                                                              \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_2 (Sl  ()                  0           ['tf.compat.v1.shape_2[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_3 (Sl  ()                  0           ['tf.compat.v1.shape_3[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.nn.relu_2 (TFOpLambda)      (None, None, 8, 64)  0           ['conv2_encoderBatchNorm[0][0]'] \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose (TFOpLa  (None, None, 64, 48  0          ['tf.nn.relu_5[0][0]']           \n",
      " mbda)                          )                                                                 \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem (Slic  ()                  0           ['tf.compat.v1.shape[0][0]']     \n",
      " ingOpLambda)                                                                                     \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_1 (Sl  ()                  0           ['tf.compat.v1.shape_1[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.math.multiply (TFOpLambda)  ()                   0           ['tf.__operators__.getitem_2[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'tf.__operators__.getitem_3[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_4 (TFOpLambda  (None, None, 10, 64  0          ['tf.nn.relu_2[0][0]']           \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " tf.reshape (TFOpLambda)        (None, None, 3072)   0           ['tf.compat.v1.transpose[0][0]', \n",
      "                                                                  'tf.__operators__.getitem[0][0]'\n",
      "                                                                 , 'tf.__operators__.getitem_1[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.math.multiply[0][0]']       \n",
      "                                                                                                  \n",
      " conv3_encoder (Conv2D)         (None, None, 8, 64)  192         ['tf.compat.v1.pad_4[0][0]']     \n",
      "                                                                                                  \n",
      " tf.split (TFOpLambda)          [(None, None, 384),  0           ['tf.reshape[0][0]']             \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384),                                               \n",
      "                                 (None, None, 384)]                                               \n",
      "                                                                                                  \n",
      " conv3_encoder_1x1 (Conv2D)     (None, None, 8, 64)  4096        ['conv3_encoder[0][0]']          \n",
      "                                                                                                  \n",
      " GFC_encoder_0 (Dense)          (None, None, 64)     24640       ['tf.split[0][0]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_1 (Dense)          (None, None, 64)     24640       ['tf.split[0][1]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_2 (Dense)          (None, None, 64)     24640       ['tf.split[0][2]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_3 (Dense)          (None, None, 64)     24640       ['tf.split[0][3]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_4 (Dense)          (None, None, 64)     24640       ['tf.split[0][4]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_5 (Dense)          (None, None, 64)     24640       ['tf.split[0][5]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_6 (Dense)          (None, None, 64)     24640       ['tf.split[0][6]']               \n",
      "                                                                                                  \n",
      " GFC_encoder_7 (Dense)          (None, None, 64)     24640       ['tf.split[0][7]']               \n",
      "                                                                                                  \n",
      " conv3_encoderBatchNorm (BatchN  (None, None, 8, 64)  256        ['conv3_encoder_1x1[0][0]']      \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " tf.stack (TFOpLambda)          (None, None, 64, 8)  0           ['GFC_encoder_0[0][0]',          \n",
      "                                                                  'GFC_encoder_1[0][0]',          \n",
      "                                                                  'GFC_encoder_2[0][0]',          \n",
      "                                                                  'GFC_encoder_3[0][0]',          \n",
      "                                                                  'GFC_encoder_4[0][0]',          \n",
      "                                                                  'GFC_encoder_5[0][0]',          \n",
      "                                                                  'GFC_encoder_6[0][0]',          \n",
      "                                                                  'GFC_encoder_7[0][0]']          \n",
      "                                                                                                  \n",
      " tf.nn.relu_3 (TFOpLambda)      (None, None, 8, 64)  0           ['conv3_encoderBatchNorm[0][0]'] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " tf.compat.v1.transpose_1 (TFOp  (None, None, 8, 64)  0          ['tf.stack[0][0]']               \n",
      " Lambda)                                                                                          \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_10 (TFOpLam  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_11 (TFOpLam  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_7 (TFOpLamb  (4,)                0           ['tf.compat.v1.transpose_1[0][0]'\n",
      " da)                                                             ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_6 (TFOpLamb  (4,)                0           ['tf.compat.v1.transpose_1[0][0]'\n",
      " da)                                                             ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_8 (TFOpLamb  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " da)                                                                                              \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_9 (TFOpLamb  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " da)                                                                                              \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_10 (S  ()                  0           ['tf.compat.v1.shape_10[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_11 (S  ()                  0           ['tf.compat.v1.shape_11[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_4 (TFOpLamb  (4,)                0           ['tf.compat.v1.transpose_1[0][0]'\n",
      " da)                                                             ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_5 (TFOpLamb  (4,)                0           ['tf.compat.v1.transpose_1[0][0]'\n",
      " da)                                                             ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_7 (Sl  ()                  0           ['tf.compat.v1.shape_7[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_6 (Sl  ()                  0           ['tf.compat.v1.shape_6[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_2 (TFOp  (None, None, 64, 8)  0          ['tf.nn.relu_3[0][0]']           \n",
      " Lambda)                                                                                          \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_8 (Sl  ()                  0           ['tf.compat.v1.shape_8[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_9 (Sl  ()                  0           ['tf.compat.v1.shape_9[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.math.multiply_2 (TFOpLambda  ()                  0           ['tf.__operators__.getitem_10[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_11[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_4 (Sl  ()                  0           ['tf.compat.v1.shape_4[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_5 (Sl  ()                  0           ['tf.compat.v1.shape_5[0][0]']   \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.math.multiply_1 (TFOpLambda  ()                  0           ['tf.__operators__.getitem_7[0][0\n",
      " )                                                               ]',                              \n",
      "                                                                  'tf.__operators__.getitem_6[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " tf.reshape_2 (TFOpLambda)      (None, None, 512)    0           ['tf.compat.v1.transpose_2[0][0]'\n",
      "                                                                 , 'tf.__operators__.getitem_8[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_9[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'tf.math.multiply_2[0][0]']     \n",
      "                                                                                                  \n",
      " tf.reshape_1 (TFOpLambda)      (None, None, 512)    0           ['tf.compat.v1.transpose_1[0][0]'\n",
      "                                                                 , 'tf.__operators__.getitem_4[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_5[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'tf.math.multiply_1[0][0]']     \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)      (None, None, 1024)   0           ['tf.reshape_2[0][0]',           \n",
      "                                                                  'tf.reshape_1[0][0]']           \n",
      "                                                                                                  \n",
      " GGRU00 (GRU)                   (None, None, 256)    984576      ['concatenate[0][0]']            \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " tf.nn.relu_6 (TFOpLambda)      (None, None, 256)    0           ['GGRU00[0][0]']                 \n",
      "                                                                                                  \n",
      " GGRU010 (GRU)                  (None, None, 256)    394752      ['tf.nn.relu_6[0][0]']           \n",
      "                                                                                                  \n",
      " tf.nn.relu_7 (TFOpLambda)      (None, None, 256)    0           ['GGRU010[0][0]']                \n",
      "                                                                                                  \n",
      " GGRU011 (GRU)                  (None, None, 256)    394752      ['tf.nn.relu_7[0][0]']           \n",
      "                                                                                                  \n",
      " tf.nn.relu_8 (TFOpLambda)      (None, None, 256)    0           ['GGRU011[0][0]']                \n",
      "                                                                                                  \n",
      " tf.__operators__.add (TFOpLamb  (None, None, 256)   0           ['tf.nn.relu_7[0][0]',           \n",
      " da)                                                              'tf.nn.relu_8[0][0]']           \n",
      "                                                                                                  \n",
      " tf.split_1 (TFOpLambda)        [(None, None, 32),   0           ['tf.__operators__.add[0][0]']   \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32)]                                                \n",
      "                                                                                                  \n",
      " GFC_decoder_0 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][0]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_1 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][1]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_2 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][2]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_3 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][3]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_4 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][4]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_5 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][5]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_6 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][6]']             \n",
      "                                                                                                  \n",
      " GFC_decoder_7 (Dense)          (None, None, 64)     2112        ['tf.split_1[0][7]']             \n",
      "                                                                                                  \n",
      " tf.stack_1 (TFOpLambda)        (None, None, 64, 8)  0           ['GFC_decoder_0[0][0]',          \n",
      "                                                                  'GFC_decoder_1[0][0]',          \n",
      "                                                                  'GFC_decoder_2[0][0]',          \n",
      "                                                                  'GFC_decoder_3[0][0]',          \n",
      "                                                                  'GFC_decoder_4[0][0]',          \n",
      "                                                                  'GFC_decoder_5[0][0]',          \n",
      "                                                                  'GFC_decoder_6[0][0]',          \n",
      "                                                                  'GFC_decoder_7[0][0]']          \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_3 (TFOp  (None, None, 8, 64)  0          ['tf.stack_1[0][0]']             \n",
      " Lambda)                                                                                          \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_15 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_3[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_14 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_3[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_12 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_3[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_13 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_3[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_15 (S  ()                  0           ['tf.compat.v1.shape_15[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_14 (S  ()                  0           ['tf.compat.v1.shape_14[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_12 (S  ()                  0           ['tf.compat.v1.shape_12[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_13 (S  ()                  0           ['tf.compat.v1.shape_13[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.math.multiply_3 (TFOpLambda  ()                  0           ['tf.__operators__.getitem_15[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_14[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.reshape_3 (TFOpLambda)      (None, None, 512)    0           ['tf.compat.v1.transpose_3[0][0]'\n",
      "                                                                 , 'tf.__operators__.getitem_12[0]\n",
      "                                                                 [0]',                            \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                  'tf.__operators__.getitem_13[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.math.multiply_3[0][0]']     \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_16 (TFOpLam  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_17 (TFOpLam  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_18 (TFOpLam  (4,)                0           ['tf.nn.relu_3[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " convp3 (Conv2D)                (None, None, 8, 64)  64          ['tf.nn.relu_3[0][0]']           \n",
      "                                                                                                  \n",
      " tf.nn.relu_9 (TFOpLambda)      (None, None, 512)    0           ['tf.reshape_3[0][0]']           \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_16 (S  ()                  0           ['tf.compat.v1.shape_16[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_17 (S  ()                  0           ['tf.compat.v1.shape_17[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_18 (S  ()                  0           ['tf.compat.v1.shape_18[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " convp3BatchNorm (BatchNormaliz  (None, None, 8, 64)  256        ['convp3[0][0]']                 \n",
      " ation)                                                                                           \n",
      "                                                                                                  \n",
      " tf.reshape_4 (TFOpLambda)      (None, None, 8, Non  0           ['tf.nn.relu_9[0][0]',           \n",
      "                                e)                                'tf.__operators__.getitem_16[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_17[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_18[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.nn.relu_10 (TFOpLambda)     (None, None, 8, 64)  0           ['convp3BatchNorm[0][0]']        \n",
      "                                                                                                  \n",
      " tf.__operators__.add_1 (TFOpLa  (None, None, 8, 64)  0          ['tf.reshape_4[0][0]',           \n",
      " mbda)                                                            'tf.nn.relu_10[0][0]']          \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_8 (TFOpLambda  (None, None, 10, 64  0          ['tf.__operators__.add_1[0][0]'] \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " convt3 (Conv2D)                (None, None, 8, 64)  192         ['tf.compat.v1.pad_8[0][0]']     \n",
      "                                                                                                  \n",
      " convt3_1x1 (Conv2D)            (None, None, 8, 64)  4096        ['convt3[0][0]']                 \n",
      "                                                                                                  \n",
      " convp2 (Conv2D)                (None, None, 8, 64)  64          ['tf.nn.relu_2[0][0]']           \n",
      "                                                                                                  \n",
      " convt3BatchNorm (BatchNormaliz  (None, None, 8, 64)  256        ['convt3_1x1[0][0]']             \n",
      " ation)                                                                                           \n",
      "                                                                                                  \n",
      " convp2BatchNorm (BatchNormaliz  (None, None, 8, 64)  256        ['convp2[0][0]']                 \n",
      " ation)                                                                                           \n",
      "                                                                                                  \n",
      " tf.nn.relu_11 (TFOpLambda)     (None, None, 8, 64)  0           ['convt3BatchNorm[0][0]']        \n",
      "                                                                                                  \n",
      " tf.nn.relu_12 (TFOpLambda)     (None, None, 8, 64)  0           ['convp2BatchNorm[0][0]']        \n",
      "                                                                                                  \n",
      " tf.__operators__.add_2 (TFOpLa  (None, None, 8, 64)  0          ['tf.nn.relu_11[0][0]',          \n",
      " mbda)                                                            'tf.nn.relu_12[0][0]']          \n",
      "                                                                                                  \n",
      " tf.split_2 (TFOpLambda)        [(None, None, 8, 1)  0           ['tf.__operators__.add_2[0][0]'] \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 8, 1)                                               \n",
      "                                , (None, None, 8, 1                                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " conv2d_transpose (Conv2DTransp  (None, None, 16, 1)  3          ['tf.split_2[0][0]']             \n",
      " ose)                                                                                             \n",
      "                                                                                                  \n",
      " conv2d_transpose_1 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][1]']             \n",
      " spose)                                                                                           \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " conv2d_transpose_2 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][2]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_3 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][3]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_4 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][4]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_5 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][5]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_6 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][6]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_7 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][7]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_8 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][8]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_9 (Conv2DTran  (None, None, 16, 1)  3          ['tf.split_2[0][9]']             \n",
      " spose)                                                                                           \n",
      "                                                                                                  \n",
      " conv2d_transpose_10 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][10]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_11 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][11]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_12 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][12]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_13 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][13]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_14 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][14]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_15 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][15]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_16 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][16]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_17 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][17]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_18 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][18]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_19 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][19]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_20 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][20]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_21 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][21]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_22 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][22]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_23 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][23]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_24 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][24]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_25 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][25]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_26 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][26]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_27 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][27]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_28 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][28]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_29 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][29]']            \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_30 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][30]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_31 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][31]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_32 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][32]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_33 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][33]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_34 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][34]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_35 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][35]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_36 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][36]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_37 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][37]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_38 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][38]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_39 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][39]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_40 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][40]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_41 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][41]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_42 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][42]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_43 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][43]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_44 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][44]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_45 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][45]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_46 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][46]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_47 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][47]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_48 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][48]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_49 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][49]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_50 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][50]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_51 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][51]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_52 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][52]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_53 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][53]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_54 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][54]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_55 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][55]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_56 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][56]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " conv2d_transpose_57 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][57]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_58 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][58]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_59 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][59]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_60 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][60]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_61 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][61]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_62 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][62]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_63 (Conv2DTra  (None, None, 16, 1)  3          ['tf.split_2[0][63]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " tf.concat (TFOpLambda)         (None, None, 16, 64  0           ['conv2d_transpose[0][0]',       \n",
      "                                )                                 'conv2d_transpose_1[0][0]',     \n",
      "                                                                  'conv2d_transpose_2[0][0]',     \n",
      "                                                                  'conv2d_transpose_3[0][0]',     \n",
      "                                                                  'conv2d_transpose_4[0][0]',     \n",
      "                                                                  'conv2d_transpose_5[0][0]',     \n",
      "                                                                  'conv2d_transpose_6[0][0]',     \n",
      "                                                                  'conv2d_transpose_7[0][0]',     \n",
      "                                                                  'conv2d_transpose_8[0][0]',     \n",
      "                                                                  'conv2d_transpose_9[0][0]',     \n",
      "                                                                  'conv2d_transpose_10[0][0]',    \n",
      "                                                                  'conv2d_transpose_11[0][0]',    \n",
      "                                                                  'conv2d_transpose_12[0][0]',    \n",
      "                                                                  'conv2d_transpose_13[0][0]',    \n",
      "                                                                  'conv2d_transpose_14[0][0]',    \n",
      "                                                                  'conv2d_transpose_15[0][0]',    \n",
      "                                                                  'conv2d_transpose_16[0][0]',    \n",
      "                                                                  'conv2d_transpose_17[0][0]',    \n",
      "                                                                  'conv2d_transpose_18[0][0]',    \n",
      "                                                                  'conv2d_transpose_19[0][0]',    \n",
      "                                                                  'conv2d_transpose_20[0][0]',    \n",
      "                                                                  'conv2d_transpose_21[0][0]',    \n",
      "                                                                  'conv2d_transpose_22[0][0]',    \n",
      "                                                                  'conv2d_transpose_23[0][0]',    \n",
      "                                                                  'conv2d_transpose_24[0][0]',    \n",
      "                                                                  'conv2d_transpose_25[0][0]',    \n",
      "                                                                  'conv2d_transpose_26[0][0]',    \n",
      "                                                                  'conv2d_transpose_27[0][0]',    \n",
      "                                                                  'conv2d_transpose_28[0][0]',    \n",
      "                                                                  'conv2d_transpose_29[0][0]',    \n",
      "                                                                  'conv2d_transpose_30[0][0]',    \n",
      "                                                                  'conv2d_transpose_31[0][0]',    \n",
      "                                                                  'conv2d_transpose_32[0][0]',    \n",
      "                                                                  'conv2d_transpose_33[0][0]',    \n",
      "                                                                  'conv2d_transpose_34[0][0]',    \n",
      "                                                                  'conv2d_transpose_35[0][0]',    \n",
      "                                                                  'conv2d_transpose_36[0][0]',    \n",
      "                                                                  'conv2d_transpose_37[0][0]',    \n",
      "                                                                  'conv2d_transpose_38[0][0]',    \n",
      "                                                                  'conv2d_transpose_39[0][0]',    \n",
      "                                                                  'conv2d_transpose_40[0][0]',    \n",
      "                                                                  'conv2d_transpose_41[0][0]',    \n",
      "                                                                  'conv2d_transpose_42[0][0]',    \n",
      "                                                                  'conv2d_transpose_43[0][0]',    \n",
      "                                                                  'conv2d_transpose_44[0][0]',    \n",
      "                                                                  'conv2d_transpose_45[0][0]',    \n",
      "                                                                  'conv2d_transpose_46[0][0]',    \n",
      "                                                                  'conv2d_transpose_47[0][0]',    \n",
      "                                                                  'conv2d_transpose_48[0][0]',    \n",
      "                                                                  'conv2d_transpose_49[0][0]',    \n",
      "                                                                  'conv2d_transpose_50[0][0]',    \n",
      "                                                                  'conv2d_transpose_51[0][0]',    \n",
      "                                                                  'conv2d_transpose_52[0][0]',    \n",
      "                                                                  'conv2d_transpose_53[0][0]',    \n",
      "                                                                  'conv2d_transpose_54[0][0]',    \n",
      "                                                                  'conv2d_transpose_55[0][0]',    \n",
      "                                                                  'conv2d_transpose_56[0][0]',    \n",
      "                                                                  'conv2d_transpose_57[0][0]',    \n",
      "                                                                  'conv2d_transpose_58[0][0]',    \n",
      "                                                                  'conv2d_transpose_59[0][0]',    \n",
      "                                                                  'conv2d_transpose_60[0][0]',    \n",
      "                                                                  'conv2d_transpose_61[0][0]',    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                  'conv2d_transpose_62[0][0]',    \n",
      "                                                                  'conv2d_transpose_63[0][0]']    \n",
      "                                                                                                  \n",
      " convt2_1x1 (Conv2D)            (None, None, 16, 64  4096        ['tf.concat[0][0]']              \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " convp1 (Conv2D)                (None, None, 16, 64  64          ['tf.nn.relu_1[0][0]']           \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " convt2BatchNorm (BatchNormaliz  (None, None, 16, 64  256        ['convt2_1x1[0][0]']             \n",
      " ation)                         )                                                                 \n",
      "                                                                                                  \n",
      " convp1BatchNorm (BatchNormaliz  (None, None, 16, 64  256        ['convp1[0][0]']                 \n",
      " ation)                         )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_13 (TFOpLambda)     (None, None, 16, 64  0           ['convt2BatchNorm[0][0]']        \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_14 (TFOpLambda)     (None, None, 16, 64  0           ['convp1BatchNorm[0][0]']        \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.__operators__.add_3 (TFOpLa  (None, None, 16, 64  0          ['tf.nn.relu_13[0][0]',          \n",
      " mbda)                          )                                 'tf.nn.relu_14[0][0]']          \n",
      "                                                                                                  \n",
      " tf.split_3 (TFOpLambda)        [(None, None, 16, 1  0           ['tf.__operators__.add_3[0][0]'] \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                ),                                                                \n",
      "                                 (None, None, 16, 1                                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " conv2d_transpose_64 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][0]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_65 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][1]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_66 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][2]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_67 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][3]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_68 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][4]']             \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_69 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][5]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_70 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][6]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_71 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][7]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_72 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][8]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_73 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][9]']             \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_74 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][10]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_75 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][11]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_76 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][12]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_77 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][13]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_78 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][14]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_79 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][15]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_80 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][16]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_81 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][17]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_82 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][18]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_83 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][19]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_84 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][20]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_85 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][21]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_86 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][22]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_87 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][23]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_88 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][24]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_89 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][25]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_90 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][26]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_91 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][27]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_92 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][28]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_93 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][29]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_94 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][30]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_95 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][31]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " conv2d_transpose_96 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][32]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_97 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][33]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_98 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][34]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_99 (Conv2DTra  (None, None, 32, 1)  3          ['tf.split_3[0][35]']            \n",
      " nspose)                                                                                          \n",
      "                                                                                                  \n",
      " conv2d_transpose_100 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][36]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_101 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][37]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_102 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][38]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_103 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][39]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_104 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][40]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_105 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][41]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_106 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][42]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_107 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][43]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_108 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][44]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_109 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][45]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_110 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][46]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_111 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][47]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_112 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][48]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_113 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][49]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_114 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][50]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_115 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][51]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_116 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][52]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_117 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][53]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_118 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][54]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_119 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][55]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_120 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][56]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_121 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][57]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_122 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][58]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_123 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][59]']            \n",
      " anspose)                                                                                         \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " conv2d_transpose_124 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][60]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_125 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][61]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_126 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][62]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " conv2d_transpose_127 (Conv2DTr  (None, None, 32, 1)  3          ['tf.split_3[0][63]']            \n",
      " anspose)                                                                                         \n",
      "                                                                                                  \n",
      " tf.concat_1 (TFOpLambda)       (None, None, 32, 64  0           ['conv2d_transpose_64[0][0]',    \n",
      "                                )                                 'conv2d_transpose_65[0][0]',    \n",
      "                                                                  'conv2d_transpose_66[0][0]',    \n",
      "                                                                  'conv2d_transpose_67[0][0]',    \n",
      "                                                                  'conv2d_transpose_68[0][0]',    \n",
      "                                                                  'conv2d_transpose_69[0][0]',    \n",
      "                                                                  'conv2d_transpose_70[0][0]',    \n",
      "                                                                  'conv2d_transpose_71[0][0]',    \n",
      "                                                                  'conv2d_transpose_72[0][0]',    \n",
      "                                                                  'conv2d_transpose_73[0][0]',    \n",
      "                                                                  'conv2d_transpose_74[0][0]',    \n",
      "                                                                  'conv2d_transpose_75[0][0]',    \n",
      "                                                                  'conv2d_transpose_76[0][0]',    \n",
      "                                                                  'conv2d_transpose_77[0][0]',    \n",
      "                                                                  'conv2d_transpose_78[0][0]',    \n",
      "                                                                  'conv2d_transpose_79[0][0]',    \n",
      "                                                                  'conv2d_transpose_80[0][0]',    \n",
      "                                                                  'conv2d_transpose_81[0][0]',    \n",
      "                                                                  'conv2d_transpose_82[0][0]',    \n",
      "                                                                  'conv2d_transpose_83[0][0]',    \n",
      "                                                                  'conv2d_transpose_84[0][0]',    \n",
      "                                                                  'conv2d_transpose_85[0][0]',    \n",
      "                                                                  'conv2d_transpose_86[0][0]',    \n",
      "                                                                  'conv2d_transpose_87[0][0]',    \n",
      "                                                                  'conv2d_transpose_88[0][0]',    \n",
      "                                                                  'conv2d_transpose_89[0][0]',    \n",
      "                                                                  'conv2d_transpose_90[0][0]',    \n",
      "                                                                  'conv2d_transpose_91[0][0]',    \n",
      "                                                                  'conv2d_transpose_92[0][0]',    \n",
      "                                                                  'conv2d_transpose_93[0][0]',    \n",
      "                                                                  'conv2d_transpose_94[0][0]',    \n",
      "                                                                  'conv2d_transpose_95[0][0]',    \n",
      "                                                                  'conv2d_transpose_96[0][0]',    \n",
      "                                                                  'conv2d_transpose_97[0][0]',    \n",
      "                                                                  'conv2d_transpose_98[0][0]',    \n",
      "                                                                  'conv2d_transpose_99[0][0]',    \n",
      "                                                                  'conv2d_transpose_100[0][0]',   \n",
      "                                                                  'conv2d_transpose_101[0][0]',   \n",
      "                                                                  'conv2d_transpose_102[0][0]',   \n",
      "                                                                  'conv2d_transpose_103[0][0]',   \n",
      "                                                                  'conv2d_transpose_104[0][0]',   \n",
      "                                                                  'conv2d_transpose_105[0][0]',   \n",
      "                                                                  'conv2d_transpose_106[0][0]',   \n",
      "                                                                  'conv2d_transpose_107[0][0]',   \n",
      "                                                                  'conv2d_transpose_108[0][0]',   \n",
      "                                                                  'conv2d_transpose_109[0][0]',   \n",
      "                                                                  'conv2d_transpose_110[0][0]',   \n",
      "                                                                  'conv2d_transpose_111[0][0]',   \n",
      "                                                                  'conv2d_transpose_112[0][0]',   \n",
      "                                                                  'conv2d_transpose_113[0][0]',   \n",
      "                                                                  'conv2d_transpose_114[0][0]',   \n",
      "                                                                  'conv2d_transpose_115[0][0]',   \n",
      "                                                                  'conv2d_transpose_116[0][0]',   \n",
      "                                                                  'conv2d_transpose_117[0][0]',   \n",
      "                                                                  'conv2d_transpose_118[0][0]',   \n",
      "                                                                  'conv2d_transpose_119[0][0]',   \n",
      "                                                                  'conv2d_transpose_120[0][0]',   \n",
      "                                                                  'conv2d_transpose_121[0][0]',   \n",
      "                                                                  'conv2d_transpose_122[0][0]',   \n",
      "                                                                  'conv2d_transpose_123[0][0]',   \n",
      "                                                                  'conv2d_transpose_124[0][0]',   \n",
      "                                                                  'conv2d_transpose_125[0][0]',   \n",
      "                                                                  'conv2d_transpose_126[0][0]',   \n",
      "                                                                  'conv2d_transpose_127[0][0]']   \n",
      "                                                                                                  \n",
      " convt1_1x1 (Conv2D)            (None, None, 32, 64  4096        ['tf.concat_1[0][0]']            \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " convp0 (Conv2D)                (None, None, 32, 64  64          ['tf.nn.relu[0][0]']             \n",
      "                                )                                                                 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " GGRU10 (GRU)                   (None, None, 256)    394752      ['tf.nn.relu_6[0][0]']           \n",
      "                                                                                                  \n",
      " convt1BatchNorm (BatchNormaliz  (None, None, 32, 64  256        ['convt1_1x1[0][0]']             \n",
      " ation)                         )                                                                 \n",
      "                                                                                                  \n",
      " convp0BatchNorm (BatchNormaliz  (None, None, 32, 64  256        ['convp0[0][0]']                 \n",
      " ation)                         )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_17 (TFOpLambda)     (None, None, 256)    0           ['GGRU10[0][0]']                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_15 (TFOpLambda)     (None, None, 32, 64  0           ['convt1BatchNorm[0][0]']        \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.nn.relu_16 (TFOpLambda)     (None, None, 32, 64  0           ['convp0BatchNorm[0][0]']        \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " GGRU11 (GRU)                   (None, None, 256)    394752      ['tf.nn.relu_17[0][0]']          \n",
      "                                                                                                  \n",
      " tf.__operators__.add_4 (TFOpLa  (None, None, 32, 64  0          ['tf.nn.relu_15[0][0]',          \n",
      " mbda)                          )                                 'tf.nn.relu_16[0][0]']          \n",
      "                                                                                                  \n",
      " tf.nn.relu_18 (TFOpLambda)     (None, None, 256)    0           ['GGRU11[0][0]']                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_9 (TFOpLambda  (None, None, 34, 64  0          ['tf.__operators__.add_4[0][0]'] \n",
      " )                              )                                                                 \n",
      "                                                                                                  \n",
      " tf.__operators__.add_5 (TFOpLa  (None, None, 256)   0           ['tf.nn.relu_17[0][0]',          \n",
      " mbda)                                                            'tf.nn.relu_18[0][0]']          \n",
      "                                                                                                  \n",
      " mask_out (Conv2D)              (None, None, 32, 1)  192         ['tf.compat.v1.pad_9[0][0]']     \n",
      "                                                                                                  \n",
      " tf.split_4 (TFOpLambda)        [(None, None, 32),   0           ['tf.__operators__.add_5[0][0]'] \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32),                                                \n",
      "                                 (None, None, 32)]                                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_20 (TFOpLam  (4,)                0           ['mask_out[0][0]']               \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_21 (TFOpLam  (4,)                0           ['mask_out[0][0]']               \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_22 (TFOpLam  (4,)                0           ['mask_out[0][0]']               \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " GFC_c_0 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][0]']             \n",
      "                                                                                                  \n",
      " GFC_c_1 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][1]']             \n",
      "                                                                                                  \n",
      " GFC_c_2 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][2]']             \n",
      "                                                                                                  \n",
      " GFC_c_3 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][3]']             \n",
      "                                                                                                  \n",
      " GFC_c_4 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][4]']             \n",
      "                                                                                                  \n",
      " GFC_c_5 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][5]']             \n",
      "                                                                                                  \n",
      " GFC_c_6 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][6]']             \n",
      "                                                                                                  \n",
      " GFC_c_7 (Dense)                (None, None, 120)    3960        ['tf.split_4[0][7]']             \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_20 (S  ()                  0           ['tf.compat.v1.shape_20[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_21 (S  ()                  0           ['tf.compat.v1.shape_21[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_22 (S  ()                  0           ['tf.compat.v1.shape_22[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.stack_2 (TFOpLambda)        (None, None, 120, 8  0           ['GFC_c_0[0][0]',                \n",
      "                                )                                 'GFC_c_1[0][0]',                \n",
      "                                                                  'GFC_c_2[0][0]',                \n",
      "                                                                  'GFC_c_3[0][0]',                \n",
      "                                                                  'GFC_c_4[0][0]',                \n",
      "                                                                  'GFC_c_5[0][0]',                \n",
      "                                                                  'GFC_c_6[0][0]',                \n",
      "                                                                  'GFC_c_7[0][0]']                \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                                  \n",
      " tf.reshape_5 (TFOpLambda)      (None, None, 32)     0           ['mask_out[0][0]',               \n",
      "                                                                  'tf.__operators__.getitem_20[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_21[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_22[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_5 (TFOp  (None, None, 8, 120  0          ['tf.stack_2[0][0]']             \n",
      " Lambda)                        )                                                                 \n",
      "                                                                                                  \n",
      " tf.math.sigmoid (TFOpLambda)   (None, None, 32)     0           ['tf.reshape_5[0][0]']           \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_27 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_5[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_26 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_5[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.linalg.matmul (TFOpLambda)  (None, None, 513)    0           ['tf.math.sigmoid[0][0]']        \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_24 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_5[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_25 (TFOpLam  (4,)                0           ['tf.compat.v1.transpose_5[0][0]'\n",
      " bda)                                                            ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_27 (S  ()                  0           ['tf.compat.v1.shape_27[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_26 (S  ()                  0           ['tf.compat.v1.shape_26[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " noisy_spec (InputLayer)        [(None, None, 513,   0           []                               \n",
      "                                2)]                                                               \n",
      "                                                                                                  \n",
      " tf.expand_dims (TFOpLambda)    (None, None, 513, 1  0           ['tf.linalg.matmul[0][0]']       \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_24 (S  ()                  0           ['tf.compat.v1.shape_24[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_25 (S  ()                  0           ['tf.compat.v1.shape_25[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.math.multiply_4 (TFOpLambda  ()                  0           ['tf.__operators__.getitem_27[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_26[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.math.multiply_5 (TFOpLambda  (None, None, 513, 2  0          ['noisy_spec[0][0]',             \n",
      " )                              )                                 'tf.expand_dims[0][0]']         \n",
      "                                                                                                  \n",
      " tf.reshape_6 (TFOpLambda)      (None, None, 960)    0           ['tf.compat.v1.transpose_5[0][0]'\n",
      "                                                                 , 'tf.__operators__.getitem_24[0]\n",
      "                                                                 [0]',                            \n",
      "                                                                  'tf.__operators__.getitem_25[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.math.multiply_4[0][0]']     \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_30 (S  (None, None, 96, 2)  0          ['tf.math.multiply_5[0][0]']     \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.math.tanh (TFOpLambda)      (None, None, 960)    0           ['tf.reshape_6[0][0]']           \n",
      "                                                                                                  \n",
      " convp_DfDecoder (Conv2D)       (None, None, 96, 10  320         ['tf.nn.relu_4[0][0]']           \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad_10 (TFOpLambd  (None, None, 96, 2)  0          ['tf.__operators__.getitem_30[0][\n",
      " a)                                                              0]']                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_28 (TFOpLam  (3,)                0           ['tf.math.tanh[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.shape_29 (TFOpLam  (3,)                0           ['tf.math.tanh[0][0]']           \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " convp_DfDecoderBatchNorm (Batc  (None, None, 96, 10  40         ['convp_DfDecoder[0][0]']        \n",
      " hNormalization)                )                                                                 \n",
      "                                                                                                  \n",
      " tf.split_5 (TFOpLambda)        [(None, None, 96, 1  0           ['tf.compat.v1.pad_10[0][0]']    \n",
      "                                ),                                                                \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                 (None, None, 96, 1                                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_28 (S  ()                  0           ['tf.compat.v1.shape_28[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_29 (S  ()                  0           ['tf.compat.v1.shape_29[0][0]']  \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.nn.relu_19 (TFOpLambda)     (None, None, 96, 10  0           ['convp_DfDecoderBatchNorm[0][0]'\n",
      "                                )                                ]                                \n",
      "                                                                                                  \n",
      " tf.image.extract_patches (TFOp  (None, None, None,   0          ['tf.split_5[0][0]']             \n",
      " Lambda)                        5)                                                                \n",
      "                                                                                                  \n",
      " tf.image.extract_patches_1 (TF  (None, None, None,   0          ['tf.split_5[0][1]']             \n",
      " OpLambda)                      5)                                                                \n",
      "                                                                                                  \n",
      " tf.reshape_7 (TFOpLambda)      (None, None, 10, 96  0           ['tf.math.tanh[0][0]',           \n",
      "                                )                                 'tf.__operators__.getitem_28[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_29[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_4 (TFOp  (None, None, 10, 96  0          ['tf.nn.relu_19[0][0]']          \n",
      " Lambda)                        )                                                                 \n",
      "                                                                                                  \n",
      " tf.expand_dims_1 (TFOpLambda)  (None, None, None,   0           ['tf.image.extract_patches[0][0]'\n",
      "                                5, 1)                            ]                                \n",
      "                                                                                                  \n",
      " tf.expand_dims_2 (TFOpLambda)  (None, None, None,   0           ['tf.image.extract_patches_1[0][0\n",
      "                                5, 1)                            ]']                              \n",
      "                                                                                                  \n",
      " tf.__operators__.add_6 (TFOpLa  (None, None, 10, 96  0          ['tf.reshape_7[0][0]',           \n",
      " mbda)                          )                                 'tf.compat.v1.transpose_4[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " tf.concat_2 (TFOpLambda)       (None, None, None,   0           ['tf.expand_dims_1[0][0]',       \n",
      "                                5, 2)                             'tf.expand_dims_2[0][0]']       \n",
      "                                                                                                  \n",
      " tf.reshape_8 (TFOpLambda)      (None, None, 5, 2,   0           ['tf.__operators__.add_6[0][0]', \n",
      "                                96)                               'tf.__operators__.getitem_28[0][\n",
      "                                                                 0]',                             \n",
      "                                                                  'tf.__operators__.getitem_29[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_7 (TFOp  (None, None, 5, Non  0          ['tf.concat_2[0][0]']            \n",
      " Lambda)                        e, 2)                                                             \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose_6 (TFOp  (None, None, 5, 96,  0          ['tf.reshape_8[0][0]']           \n",
      " Lambda)                         2)                                                               \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_31 (S  (None, None, 5, Non  0          ['tf.compat.v1.transpose_7[0][0]'\n",
      " licingOpLambda)                e)                               ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_32 (S  (None, None, 5, 96)  0          ['tf.compat.v1.transpose_6[0][0]'\n",
      " licingOpLambda)                                                 ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_33 (S  (None, None, 5, Non  0          ['tf.compat.v1.transpose_7[0][0]'\n",
      " licingOpLambda)                e)                               ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_34 (S  (None, None, 5, 96)  0          ['tf.compat.v1.transpose_6[0][0]'\n",
      " licingOpLambda)                                                 ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_35 (S  (None, None, 5, Non  0          ['tf.compat.v1.transpose_7[0][0]'\n",
      " licingOpLambda)                e)                               ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_36 (S  (None, None, 5, 96)  0          ['tf.compat.v1.transpose_6[0][0]'\n",
      " licingOpLambda)                                                 ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_37 (S  (None, None, 5, Non  0          ['tf.compat.v1.transpose_7[0][0]'\n",
      " licingOpLambda)                e)                               ]                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_38 (S  (None, None, 5, 96)  0          ['tf.compat.v1.transpose_6[0][0]'\n",
      " licingOpLambda)                                                 ]                                \n",
      "                                                                                                  \n",
      " tf.math.multiply_6 (TFOpLambda  (None, None, 5, 96)  0          ['tf.__operators__.getitem_31[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_32[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.math.multiply_7 (TFOpLambda  (None, None, 5, 96)  0          ['tf.__operators__.getitem_33[0][\n",
      " )                                                               0]',                             \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                  'tf.__operators__.getitem_34[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.math.multiply_8 (TFOpLambda  (None, None, 5, 96)  0          ['tf.__operators__.getitem_35[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_36[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.math.multiply_9 (TFOpLambda  (None, None, 5, 96)  0          ['tf.__operators__.getitem_37[0][\n",
      " )                                                               0]',                             \n",
      "                                                                  'tf.__operators__.getitem_38[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " tf.math.subtract (TFOpLambda)  (None, None, 5, 96)  0           ['tf.math.multiply_6[0][0]',     \n",
      "                                                                  'tf.math.multiply_7[0][0]']     \n",
      "                                                                                                  \n",
      " tf.__operators__.add_7 (TFOpLa  (None, None, 5, 96)  0          ['tf.math.multiply_8[0][0]',     \n",
      " mbda)                                                            'tf.math.multiply_9[0][0]']     \n",
      "                                                                                                  \n",
      " tf.expand_dims_3 (TFOpLambda)  (None, None, 5, 96,  0           ['tf.math.subtract[0][0]']       \n",
      "                                 1)                                                               \n",
      "                                                                                                  \n",
      " tf.expand_dims_4 (TFOpLambda)  (None, None, 5, 96,  0           ['tf.__operators__.add_7[0][0]'] \n",
      "                                 1)                                                               \n",
      "                                                                                                  \n",
      " convp_alpha (Dense)            (None, None, 1)      257         ['tf.__operators__.add_5[0][0]'] \n",
      "                                                                                                  \n",
      " tf.concat_3 (TFOpLambda)       (None, None, 5, 96,  0           ['tf.expand_dims_3[0][0]',       \n",
      "                                 2)                               'tf.expand_dims_4[0][0]']       \n",
      "                                                                                                  \n",
      " tf.expand_dims_5 (TFOpLambda)  (None, None, 1, 1)   0           ['convp_alpha[0][0]']            \n",
      "                                                                                                  \n",
      " tf.math.reduce_sum (TFOpLambda  (None, None, 96, 2)  0          ['tf.concat_3[0][0]']            \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_39 (S  (None, None, 96, 2)  0          ['tf.math.multiply_5[0][0]']     \n",
      " licingOpLambda)                                                                                  \n",
      "                                                                                                  \n",
      " tf.math.subtract_1 (TFOpLambda  (None, None, 1, 1)  0           ['tf.expand_dims_5[0][0]']       \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " tf.math.multiply_10 (TFOpLambd  (None, None, 96, 2)  0          ['tf.math.reduce_sum[0][0]',     \n",
      " a)                                                               'tf.expand_dims_5[0][0]']       \n",
      "                                                                                                  \n",
      " tf.math.multiply_11 (TFOpLambd  (None, None, 96, 2)  0          ['tf.__operators__.getitem_39[0][\n",
      " a)                                                              0]',                             \n",
      "                                                                  'tf.math.subtract_1[0][0]']     \n",
      "                                                                                                  \n",
      " tf.__operators__.add_8 (TFOpLa  (None, None, 96, 2)  0          ['tf.math.multiply_10[0][0]',    \n",
      " mbda)                                                            'tf.math.multiply_11[0][0]']    \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_40 (S  (None, None, 417, 2  0          ['tf.math.multiply_5[0][0]']     \n",
      " licingOpLambda)                )                                                                 \n",
      "                                                                                                  \n",
      " clean_spec (InputLayer)        [(None, None, 513,   0           []                               \n",
      "                                2)]                                                               \n",
      "                                                                                                  \n",
      " tf.concat_4 (TFOpLambda)       (None, None, 513, 2  0           ['tf.__operators__.add_8[0][0]', \n",
      "                                )                                 'tf.__operators__.getitem_40[0][\n",
      "                                                                 0]']                             \n",
      "                                                                                                  \n",
      " maskloss (Lambda)              ()                   0           ['tf.math.sigmoid[0][0]',        \n",
      "                                                                  'clean_spec[0][0]',             \n",
      "                                                                  'noisy_spec[0][0]']             \n",
      "                                                                                                  \n",
      " spectralloss (Lambda)          ()                   0           ['tf.concat_4[0][0]',            \n",
      "                                                                  'clean_spec[0][0]']             \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 2,848,937\n",
      "Trainable params: 2,847,253\n",
      "Non-trainable params: 1,684\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2cf44364",
   "metadata": {},
   "outputs": [],
   "source": [
    "# warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base=p.lr,\n",
    "#                                         total_steps=p.epochs*steps_per_epoch,\n",
    "#                                         warmup_steps=(p.epochs//10)*steps_per_epoch,\n",
    "#                                         warmup_learning_rate = 1e-4,\n",
    "#                                         mini_lr=1e-7,\n",
    "#                                         steps_per_epoch=steps_per_epoch,\n",
    "#                                         global_step_init=initial_epoch*steps_per_epoch)\n",
    "warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base=p.lr,\n",
    "                                        total_steps=p.epochs*steps_per_epoch,\n",
    "                                        warmup_steps=0*steps_per_epoch,\n",
    "                                        warmup_learning_rate = 1e-4,\n",
    "                                        mini_lr=1e-7,\n",
    "                                        steps_per_epoch=steps_per_epoch,\n",
    "                                        global_step_init=initial_epoch*steps_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "851a5183",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt_lr = []\n",
    "# for i in range(p.epochs):\n",
    "#     plt_lr.append(cosine_decay_with_warmup(global_step=i,\n",
    "#                               learning_rate_base=1e-3,\n",
    "#                               total_steps=30*1,\n",
    "#                               warmup_learning_rate=1e-4,\n",
    "#                               warmup_steps=3*1,\n",
    "#                               hold_base_rate_steps=0,\n",
    "#                               steps_per_epoch=1))\n",
    "# for i in range(3):\n",
    "#     for j in range(10):\n",
    "#         print('%.7f'%plt_lr[i*10+j], end=' ')\n",
    "#     print()\n",
    "# plt_lr[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c8f9e808",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "cp_callback = ModelCheckpoint(model, './weights8/weights_newlossadam.{epoch:02d}-{loss:.2f}.h5')\n",
    "callbacks_overall = [cp_callback, callbacks_nan]#, warm_up_lr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "93303658",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add_metric(maskloss, name = \"maskloss\")\n",
    "model.add_metric(spectralloss, name=\"spectralloss\")\n",
    "# model.add_metric(spectralloss2, name=\"spectralloss2\")\n",
    "# model.add_loss(maskloss)\n",
    "model.add_loss(spectralloss)\n",
    "# model.add_loss(spectralloss2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7408e444",
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizer = AdamW(learning_rate=p.lr, weight_decay=0.0, clipnorm=1.0)\n",
    "# optimizer = Adam(learning_rate=p.lr, clipnorm=1.0)\n",
    "optimizer = SGD(learning_rate=p.lr, clipnorm=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "579ed691",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=optimizer, run_eagerly=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "227dc812",
   "metadata": {},
   "source": [
    "# Load weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "491fc917",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./weights8/weights_newlossadam.38-75.06.h5\n"
     ]
    }
   ],
   "source": [
    "path = \"./weights8/\"\n",
    "h5_lists = os.listdir(path)\n",
    "h5_lists.sort(key=lambda fn:os.path.getmtime(path + fn)\n",
    "                if not os.path.isdir(path + fn) else 0)\n",
    "new_file = os.path.join(path, h5_lists[-1])\n",
    "# new_file = \"weights.19-58.49.h5\"\n",
    "print(new_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3af2640e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/myhsueh/.local/lib/python3.8/site-packages/numpy/core/numeric.py:2443: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  a1, a2 = asarray(a1), asarray(a2)\n"
     ]
    }
   ],
   "source": [
    "# store weights before loading pre-trained weights\n",
    "preloaded_layers = model.layers.copy()\n",
    "preloaded_weights = []\n",
    "for pre in preloaded_layers:\n",
    "    preloaded_weights.append(pre.get_weights())\n",
    "    \n",
    "# load pre-trained weights\n",
    "model.load_weights(new_file, by_name=True)\n",
    "\n",
    "# compare previews weights vs loaded weights\n",
    "for layer, pre in zip(model.layers, preloaded_weights):\n",
    "    weights = layer.get_weights()\n",
    "\n",
    "    if weights:\n",
    "        if np.array_equal(weights, pre):\n",
    "            print('not loaded', layer.name)\n",
    "        else:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c39217d1",
   "metadata": {},
   "source": [
    "# TRAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0194022e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.3522 - maskloss: 7.5296 - spectralloss: 70.3522\n",
      "Saving model to ./weights8/weights_newlossadam.39-75.52.h5\n",
      ", Validation loss decreased from inf to 75.51563262939453.\n",
      "\n",
      "7609/7609 [==============================] - 4639s 606ms/step - loss: 70.3522 - maskloss: 7.5296 - spectralloss: 70.3522 - val_loss: 75.5156 - val_maskloss: 10.9232 - val_spectralloss: 75.5156\n",
      "Epoch 41/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.4884 - maskloss: 8.1881 - spectralloss: 70.4884\n",
      "Saving model to ./weights8/weights_newlossadam.40-75.38.h5\n",
      ", Validation loss decreased from 75.51563262939453 to 75.38206481933594.\n",
      "\n",
      "7609/7609 [==============================] - 4739s 623ms/step - loss: 70.4884 - maskloss: 8.1881 - spectralloss: 70.4884 - val_loss: 75.3821 - val_maskloss: 11.2266 - val_spectralloss: 75.3821\n",
      "Epoch 42/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.4197 - maskloss: 8.4398 - spectralloss: 70.4197\n",
      "Saving model to ./weights8/weights_newlossadam.41-75.25.h5\n",
      ", Validation loss decreased from 75.38206481933594 to 75.24954986572266.\n",
      "\n",
      "7609/7609 [==============================] - 4422s 581ms/step - loss: 70.4197 - maskloss: 8.4398 - spectralloss: 70.4197 - val_loss: 75.2495 - val_maskloss: 11.2538 - val_spectralloss: 75.2495\n",
      "Epoch 43/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.3606 - maskloss: 8.5233 - spectralloss: 70.3606\n",
      "Saving model to ./weights8/weights_newlossadam.42-75.16.h5\n",
      ", Validation loss decreased from 75.24954986572266 to 75.16417694091797.\n",
      "\n",
      "7609/7609 [==============================] - 4453s 585ms/step - loss: 70.3606 - maskloss: 8.5233 - spectralloss: 70.3606 - val_loss: 75.1642 - val_maskloss: 11.3031 - val_spectralloss: 75.1642\n",
      "Epoch 44/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.3474 - maskloss: 8.5897 - spectralloss: 70.3474\n",
      "Saving model to ./weights8/weights_newlossadam.43-75.08.h5\n",
      ", Validation loss decreased from 75.16417694091797 to 75.07879638671875.\n",
      "\n",
      "7609/7609 [==============================] - 4534s 596ms/step - loss: 70.3474 - maskloss: 8.5897 - spectralloss: 70.3474 - val_loss: 75.0788 - val_maskloss: 11.3134 - val_spectralloss: 75.0788\n",
      "Epoch 45/50\n",
      "7609/7609 [==============================] - ETA: 0s - loss: 70.2261 - maskloss: 8.6638 - spectralloss: 70.2261\n",
      "Saving model to ./weights8/weights_newlossadam.44-75.02.h5\n",
      ", Validation loss decreased from 75.07879638671875 to 75.0167007446289.\n",
      "\n",
      "7609/7609 [==============================] - 4617s 607ms/step - loss: 70.2261 - maskloss: 8.6638 - spectralloss: 70.2261 - val_loss: 75.0167 - val_maskloss: 11.3529 - val_spectralloss: 75.0167\n",
      "Epoch 46/50\n",
      "4941/7609 [==================>...........] - ETA: 23:53 - loss: 80.0524 - maskloss: 8.9555 - spectralloss: 80.0524"
     ]
    }
   ],
   "source": [
    "history = model.fit(dataset,\n",
    "                    epochs=p.epochs,\n",
    "                    validation_data=val_dataset,\n",
    "                    steps_per_epoch=steps_per_epoch,\n",
    "                    validation_steps=validation_steps,\n",
    "                    callbacks=callbacks_overall,\n",
    "                    initial_epoch=initial_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d242bc54",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
