{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7d61fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import autoreload\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d918d51f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import random\n",
    "import os\n",
    "# from scipy.io.wavfile import read, write\n",
    "# from scipy.signal import resample\n",
    "from librosa import resample, load\n",
    "from soundfile import write\n",
    "import logging\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "de6e0c23",
   "metadata": {},
   "outputs": [],
   "source": [
    "# logging.getLogger().setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ddcecf32",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2bc3faff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from params.ipynb\n",
      "importing Jupyter notebook from bandERB.ipynb\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "from params import model_params\n",
    "from bandERB import ERBBand, ERB_pro_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1e36333b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mixkwargs={\n",
    "    \"dataset_path\": '/home3/user/myhsueh/',\n",
    "    \"speech_txt\": '/home3/user/myhsueh/training_set_speech.txt',\n",
    "    \"noise_txt\": '/home3/user/myhsueh/training_set_noise.txt',\n",
    "#     \"noise_txt\": '/home3/user/myhsueh/training_set_noise_rare2.txt',\n",
    "#     \"noise_txt\": '/home3/user/myhsueh/training_set_noise_kb.txt',\n",
    "    \"rir_txt\": '/home3/user/myhsueh/training_set_rir.txt',\n",
    "    \"RIR\": False,\n",
    "    \"multi_noise\": False,\n",
    "    \"downsample\": True,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "93331585",
   "metadata": {},
   "outputs": [],
   "source": [
    "def type_check(y):\n",
    "    if y.dtype == 'int32': y = y / (2**31)\n",
    "    elif y.dtype == 'int16': my = y / (2**15)\n",
    "    elif y.dtype == 'uint8': y = y / (2**7)\n",
    "    elif y.dtype == 'float32': pass\n",
    "    else: \n",
    "        print('Other, data type:', y.dtype)\n",
    "    return y\n",
    "    \n",
    "def sr_check(y, sr, target_sr = 48000):\n",
    "    length = y.shape[0]\n",
    "    if not sr == target_sr:\n",
    "#         new_length = int(length / sr * target_sr)\n",
    "        new_y = resample(y, sr, target_sr)\n",
    "        logging.info('Change sampling rate from %d to %d'%(sr, target_sr))\n",
    "    else:\n",
    "        new_y = y\n",
    "    return target_sr, new_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "61e0164a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Augmentation_tool(dataset_path, speech_txt, noise_txt, \n",
    "#                       speech_id, noise_id, \n",
    "                      rir_txt=None, target_sr = 48000, \n",
    "                      RIR=True, multi_noise=False, random_delay=True,\n",
    "                      speech_gain_dB=[-6,6], SNRs_dB=[-10,-5,0,5,10,20,40,45], downsample=True):\n",
    "   \n",
    "    speech_list = open(speech_txt).readlines() \n",
    "    noise_list = open(noise_txt).readlines() \n",
    "    num_of_speech = len(speech_list) \n",
    "    num_of_noise = len(noise_list) \n",
    "    if RIR: \n",
    "        if os.path.isfile(rir_txt):\n",
    "            rir_list = open(rir_txt).readlines() \n",
    "            num_of_rir = len(rir_list) \n",
    "        else:\n",
    "            RIR = False\n",
    "######################################################################\n",
    "##                         Speech selection                         ## \n",
    "######################################################################\n",
    "    speech_idx = random.randint(0,num_of_speech-1)\n",
    "#     speech_idx = speech_id % num_of_speech\n",
    "    speech_filename = os.path.join(dataset_path, speech_list[speech_idx].rstrip('\\n'))\n",
    "    \n",
    "    y, sr = load(speech_filename, sr=None)\n",
    "    sr, y = sr_check(y, sr)\n",
    "    y = type_check(y)\n",
    "    speech_length = y.shape[0]  \n",
    "    while speech_length<sr*3:\n",
    "        speech_idx = random.randint(0,num_of_speech-1)\n",
    "        speech_filename = os.path.join(dataset_path, speech_list[speech_idx].rstrip('\\n'))\n",
    "\n",
    "        y, sr = load(speech_filename, sr=None)\n",
    "        sr, y = sr_check(y, sr)\n",
    "        y = type_check(y)\n",
    "        speech_length = y.shape[0]  \n",
    "    logging.info('[SPEECH] DONE!')\n",
    "    \n",
    "######################################################################\n",
    "##                         Noise collection                         ##\n",
    "######################################################################\n",
    "    noise_idx = random.randint(0,num_of_noise-1)\n",
    "#     noise_idx = noise_id % num_of_noise\n",
    "    noise_filename = os.path.join(dataset_path, noise_list[noise_idx].rstrip('\\n'))\n",
    "    \n",
    "    noise, noise_sr = load(noise_filename, sr=None)\n",
    "    noise_sr, noise = sr_check(noise, noise_sr)\n",
    "    noise = type_check(noise)\n",
    "    noise_length = noise.shape[0]\n",
    "#     while noise_length<sr*3:\n",
    "#         noise_idx = random.randint(0,num_of_noise-1)\n",
    "#         noise_filename = os.path.join(dataset_path, noise_list[noise_idx].rstrip('\\n'))\n",
    "\n",
    "#         noise, noise_sr = load(noise_filename, sr=None)\n",
    "#         noise_sr, noise = sr_check(noise, noise_sr)\n",
    "#         noise = type_check(noise)\n",
    "#         noise_length = noise.shape[0]\n",
    "#     if multi_noise:\n",
    "#         num_noise = random.randint(2,4)\n",
    "#         for _ in range(1,num_noise):\n",
    "#             noise_idx2 = random.randint(0,num_of_noise-1)\n",
    "#             while noise_idx2 == noise_idx:\n",
    "#                 noise_idx2 = random.randint(0,num_of_noise-1)\n",
    "#             noise_filename2 = os.path.join(dataset_path, noise_list[noise_idx2].rstrip('\\n'))\n",
    "\n",
    "#             noise2, noise_sr2 = load(noise_filename2, sr=None)\n",
    "#             noise_sr2, noise2 = sr_check(noise2, noise_sr2)\n",
    "#             noise2 = type_check(noise2)\n",
    "#             noise_length2 = noise2.shape[0]\n",
    "#             if noise_length2>noise_length:\n",
    "#                 start_pad = random.randint(0,noise_length2-noise_length)\n",
    "#                 end_pad = noise_length2 - noise_length - start_pad\n",
    "#                 noise = np.pad(noise,(start_pad,end_pad))\n",
    "#             elif noise_length2<noise_length:\n",
    "#                 start_pad = random.randint(0,noise_length-noise_length2)\n",
    "#                 end_pad = noise_length - noise_length2 - start_pad\n",
    "#                 noise2 = np.pad(noise2,(start_pad,end_pad))\n",
    "#             else:\n",
    "#                 pass\n",
    "#             noise = noise + noise2\n",
    "#             noise_length = noise.shape[0]\n",
    "            \n",
    "    logging.info('[NOISE] DONE!')\n",
    "\n",
    "######################################################################\n",
    "##                             Add RIR                              ##\n",
    "######################################################################\n",
    "    if RIR and (random.randint(0,2) == 0):\n",
    "        # random select RIR files\n",
    "        rir_idx = random.randint(0,num_of_rir-1)\n",
    "        rir_filename = os.path.join(dataset_path, rir_list[rir_idx].rstrip('\\n'))\n",
    "        # read RIR file\n",
    "        rir, rir_sr = load(rir_filename, sr=None)\n",
    "        rir_sr, rir = sr_check(rir, rir_sr)\n",
    "        rir = type_check(rir)\n",
    "        if max(abs(rir))>1:\n",
    "            rir /=  (max(abs(rir))+1e-10)\n",
    "        rir_length = rir.shape[0]\n",
    "        if rir_length > target_sr:\n",
    "            rir = rir[:target_sr]\n",
    "            \n",
    "        random_num = random.randint(0,2)\n",
    "        if  random_num % 2 == 0:\n",
    "            # convolve with RIR\n",
    "            y_rir = np.convolve(y, rir, mode='same')\n",
    "\n",
    "            target_rir = rir_attenuation(rir)\n",
    "            target_y = np.convolve(y, target_rir, mode='same')\n",
    "            if random_num == 0:\n",
    "                noise = np.convolve(noise, rir, mode='same')\n",
    "        else:\n",
    "            noise = np.convolve(noise, rir, mode='same')\n",
    "            y_rir = y\n",
    "            target_y = y\n",
    "            \n",
    "        logging.info('[RIR] DONE!')    \n",
    "    else: \n",
    "        y_rir = y\n",
    "        target_y = y\n",
    "        logging.info('[RIR] NO file') \n",
    "        \n",
    "######################################################################\n",
    "##                           Apply Gain                             ##\n",
    "######################################################################\n",
    "  \n",
    "    if y_rir.shape[0]>noise.shape[0]:\n",
    "        start_pad = random.randint(0,y_rir.shape[0]-noise.shape[0])\n",
    "        end_pad = y_rir.shape[0] - noise.shape[0] - start_pad\n",
    "        noise = np.pad(noise,(start_pad,end_pad))\n",
    "    else:\n",
    "        start_pad = random.randint(0,noise.shape[0]-y_rir.shape[0])\n",
    "        noise = noise[start_pad:start_pad+y_rir.shape[0]]\n",
    "\n",
    "    gain_dB = random.uniform(-6,6)\n",
    "    gain_linear = 10**(gain_dB/20)\n",
    "    y_rir *= gain_linear\n",
    "    target_y *= gain_linear\n",
    "    \n",
    "    SNR = random.choice(SNRs_dB)\n",
    "    SNR_linear = 10**(SNR/20)\n",
    "    noise_gain = compute_SNR_gain(y_rir, noise, SNR_linear)\n",
    "    noise *= noise_gain\n",
    "    \n",
    "    logging.info('[Gain] Speech: %.2f, Noise: %.2f, SNR(dB): %.2f'%(gain_linear,noise_gain,SNR))\n",
    "        \n",
    "    mixture = y_rir + noise\n",
    "\n",
    "######################################################################\n",
    "##                         Random downsample                        ##                              \n",
    "######################################################################\n",
    "    if downsample and (random.randint(0,10)<3):  # down to 16k and upsample to 48k\n",
    "        random_cutoff = (random.randrange(6, 48, 2)) * 1000\n",
    "        # Downsample\n",
    "        target_sr, mixture_down = sr_check(mixture, sr, target_sr=random_cutoff)\n",
    "        target_sr, target_y_down = sr_check(target_y, sr, target_sr=random_cutoff)\n",
    "        # Upsample to 48\n",
    "        _, mixture = sr_check(mixture_down, sr=target_sr)\n",
    "        _, target_y = sr_check(target_y_down, sr=target_sr)\n",
    "    else:\n",
    "        random_cutoff = 48000\n",
    "\n",
    "######################################################################\n",
    "##                          Max value clip                          ##\n",
    "######################################################################       \n",
    "    max_val = max(max(abs(mixture)), max(abs(target_y)))\n",
    "    \n",
    "    if max_val>1:\n",
    "        mixture /= (max_val + 1e-10)\n",
    "        target_y /= (max_val +1e-10)\n",
    "        logging.info('[MIXTURE] Normalized from %f.'%(max_val))\n",
    "    \n",
    "    return mixture, target_y\n",
    "# , speech_id+1, noise_id+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a4280d87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_SNR_gain(y, noise, SNR_linear):\n",
    "    speech_power = np.mean(np.abs(y))\n",
    "    noise_power = np.mean(np.abs(noise))\n",
    "\n",
    "    snr = (speech_power / (noise_power + 10**-6))\n",
    "    noise_gain = snr / SNR_linear\n",
    "    \n",
    "    return noise_gain\n",
    "\n",
    "def rir_attenuation(rir, offset=5, target_sr=48000):\n",
    "    peak_idx = np.argmax(np.abs(rir))\n",
    "    peak_idx += int((5/1000)*target_sr)\n",
    "    weights = np.ones((rir.shape[0],))\n",
    "    \n",
    "    for i in range(rir.shape[0]-peak_idx):\n",
    "        weights[i+peak_idx] = np.exp(-i/target_sr/(-0.2/np.log10(10**-3)))\n",
    "    return np.multiply(rir, weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a24d976b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mixture, target_y = Augmentation_tool(**mixkwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1dcffd94",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import tensorflow as tf\n",
    "def vorbis_window(FRAME_SIZE, transpose=True):\n",
    "    FRAME_SIZE = FRAME_SIZE//2\n",
    "    win = np.zeros((FRAME_SIZE,))\n",
    "    for i in range(FRAME_SIZE):\n",
    "        win[i] = np.sin(.5*np.pi*np.sin(.5*np.pi*(i+.5)/FRAME_SIZE) * np.sin(.5*np.pi*(i+.5)/FRAME_SIZE))\n",
    "    win = np.concatenate((win,np.flip(win)),0)\n",
    "    if transpose: win = win.T\n",
    "    return win"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0cbd74d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analysis_frame(x, nfft=960, hop=480, normalize=False):\n",
    "    length = len(x)\n",
    "    n_frames = length // hop\n",
    "    out = np.empty((n_frames, nfft//2+1),dtype=complex)\n",
    "    if not length % hop == 0:\n",
    "        x = np.pad(x,(0, nfft - length%hop))\n",
    "    for frame_idx in range(0, n_frames * hop, hop):\n",
    "        frame = x[frame_idx : frame_idx + nfft]\n",
    "        if len(frame)<nfft: frame = np.pad(frame,(0,nfft-len(frame)))\n",
    "#         win = np.hanning(nfft)\n",
    "        win = vorbis_window(nfft)\n",
    "        frame = frame.reshape(win.shape)\n",
    "        frame_win = np.multiply(frame, win)\n",
    "        x_fft = np.fft.rfft(frame_win, n=p.fft_size) \n",
    "        if normalize: x_fft * (p.fft_size ** -0.5)\n",
    "        out[frame_idx//hop,:] = x_fft\n",
    "    return out "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6281426a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7f3a553b",
   "metadata": {},
   "outputs": [],
   "source": [
    "length = 48000*3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f0a122d",
   "metadata": {},
   "outputs": [],
   "source": [
    "mother_path = '/home3/user/myhsueh/h5_dataset/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "548593a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = model_params('config.ini')\n",
    "ERBB = ERBBand(N=p.nb_erb, high_lim=p.sr//2, NFFT=p.fft_size)\n",
    "ERB_Matrix = ERB_pro_matrix(ERBB, NFFT=p.fft_size, mode=0)\n",
    "iERB_Matrix = ERB_pro_matrix(ERBB, NFFT=p.fft_size, mode=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "28d67493",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(file_idx=0):\n",
    "#     speech_id = file_idx * 670\n",
    "#     noise_id = speech_id\n",
    "    while True:\n",
    "        h5f_clean_td = h5py.File(mother_path + '/TD_' + str(file_idx) + '_clean.h5', 'a')\n",
    "        h5f_noisy_td = h5py.File(mother_path + '/TD_' + str(file_idx) + '_noisy.h5', 'a')\n",
    "        \n",
    "        mixture, target_y = Augmentation_tool(\n",
    "#             speech_id=speech_id, noise_id=noise_id, \n",
    "            **mixkwargs) # time domain\n",
    "        \n",
    "        data_clean_td = np.empty((target_y.shape[0]//length, length))\n",
    "        data_noisy_td = np.empty((mixture.shape[0]//length, length))\n",
    "        for j in range(target_y.shape[0]//length):\n",
    "            data_clean_td[j] = target_y[j*length:(j+1)*length]\n",
    "            data_noisy_td[j] = mixture[j*length:(j+1)*length]\n",
    "        \n",
    "        idx_add = save_h5(h5f_clean_td,np.array(data_clean_td),'data',max_len=6000//3, flag=True)\n",
    "        save_h5(h5f_noisy_td,np.array(data_noisy_td),'data',max_len=6000//3)\n",
    "\n",
    "        h5f_clean_td.close()\n",
    "        h5f_noisy_td.close()\n",
    "        \n",
    "        if idx_add == True: break\n",
    "#     return speech_id, noise_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7e9175f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def generator_audio(times=1):\n",
    "#     for i in range(times):\n",
    "#         mixture, target_y, cutoff = Augmentation_tool(**mixkwargs) # time domain\n",
    "#         print('write', i)\n",
    "#         print(mixture.shape)\n",
    "#         print(target_y.shape)\n",
    "#         write('/home3/user/myhsueh/augment_' + str(i) + '.wav', \n",
    "#               np.concatenate((np.reshape(mixture,(mixture.shape[0], 1)),\n",
    "#                               np.reshape(target_y,(target_y.shape[0],1))),1), 48000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a1548a31",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_h5(h5f,data,target,max_len,flag=False):\n",
    "    shape_list=list(data.shape)\n",
    "    if not h5f.__contains__(target):\n",
    "        shape_list[0]=None\n",
    "        dataset = h5f.create_dataset(target, data=data, maxshape=tuple(shape_list), chunks=True)\n",
    "        return\n",
    "    else:\n",
    "        dataset = h5f[target]\n",
    "    len_old=dataset.shape[0]\n",
    "    len_new=len_old+data.shape[0]\n",
    "    if len_old>=max_len:\n",
    "        pass\n",
    "    else:\n",
    "        if len_new>=max_len: \n",
    "            len_new = max_len\n",
    "            data = data[:len_new-len_old]\n",
    "        else:\n",
    "            data = data\n",
    "        shape_list[0]=len_new\n",
    "        dataset.resize(tuple(shape_list))\n",
    "        dataset[len_old:] = data\n",
    "    \n",
    "    if flag: \n",
    "        if len_new>=max_len: \n",
    "            return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6d20a70d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(mother_path): \n",
    "    os.makedirs(mother_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46a6ee8f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "db30ec94",
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9983be01",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threads = mp.cpu_count()\n",
    "pool = mp.Pool(processes=10)\n",
    "pool.map(generator, range(70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5547be46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# speech_id, noise_id = 0, 0\n",
    "# for i in range(100):\n",
    "#     speech_id, noise_id = generator(i, speech_id, noise_id)\n",
    "#     print(\"i:%d, speech_id: %d, noise_id:%d \\n\" %(i, speech_id, noise_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "29cf165c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with h5py.File('/home3/user/myhsueh/h5_dataset/3.h5', 'r') as hf:\n",
    "#     data = hf['data'][:]\n",
    "# data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d4f0dc71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file directory exist: True\n",
      "['/home3/user/myhsueh/h5_dataset/TD_26_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_31_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_74KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_22_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_20_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_24_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_85KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_41_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_71KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_78KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_23_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_19_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_81KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_47_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_88KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_82KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_87KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_86KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_44_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_60_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_90KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_56_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_35_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_75KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_58_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_33_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_1_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_8_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_96KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_9_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_98KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_11_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_92KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_38_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_13_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_70KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_4_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_7_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_66_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_40_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_16_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_3_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_14_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_62_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_39_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_6_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_61_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_10_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_64_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_52_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_17_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_69_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_30_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_80KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_93KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_45_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_28_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_65_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_95KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_29_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_46_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_37_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_72KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_59_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_76KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_32_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_43_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_34_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_83KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_63_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_21_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_51_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_99KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_25_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_48_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_42_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_50_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_79KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_84KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_91KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_55_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_94KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_49_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_73KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_53_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_18_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_97KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_89KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_54_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_5_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_0_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_68_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_27_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_15_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_2_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_12_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_57_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_67_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_77KB_clean.h5', '/home3/user/myhsueh/h5_dataset/TD_36_clean.h5']\n",
      "100\n",
      "\n",
      "Start generate TD_70KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_70KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_70KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_70KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_12-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_12-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_12-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_12-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_24-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_0-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_26-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_24-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_0-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_32-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_16-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_26-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_24-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_0-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_32-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_16-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_26-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_24-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_0-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_6-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_32-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_26-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_6-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_32-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_6-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_6-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_82KB-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_82KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_18-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_82KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_21-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_18-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_82KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_21-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_42-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_18-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_21-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_42-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_18-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_21-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_42-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_42-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_23-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_33-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_23-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_33-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_23-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_46-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_33-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_23-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_46-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_33-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_46-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_46-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_17-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_17-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_17-3-of-4.tfrecord\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start generate TD_17-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_90KB-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_90KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_45-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_90KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_45-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_90KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_45-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_45-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_55-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_68-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_55-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_68-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_55-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_68-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_55-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_68-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_57-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_85KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_57-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_85KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_57-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_85KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_57-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_85KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_97KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_97KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_31-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_97KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_31-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_97KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_31-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_31-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_61-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_61-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_87KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_61-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_87KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_61-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_87KB-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_87KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_1-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_27-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_1-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_4-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_27-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_1-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_51-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_4-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_27-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_1-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_69-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_94KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_51-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_4-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_27-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_69-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_28-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_19-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_94KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_51-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_4-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_67-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_69-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_28-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_19-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_94KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_51-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_67-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_69-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_28-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_19-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_94KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_67-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_28-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_19-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_56-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_67-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_56-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_37-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_56-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_37-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_56-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_37-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_37-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_74KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_74KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_74KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_74KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_86KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_86KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_86KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_86KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_10-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_10-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_10-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_10-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_8-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_8-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_8-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_30-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_8-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_30-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_7-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_30-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_7-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_30-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_7-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_7-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_99KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_41-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_99KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_22-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_41-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_99KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_22-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_41-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_49-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_99KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_89KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_22-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_41-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_81KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_49-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_89KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_22-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_81KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_49-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_89KB-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_81KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_72KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_49-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_89KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_81KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_72KB-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_72KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_72KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_35-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_35-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_35-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_35-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_65-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_65-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_65-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_65-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_15-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_15-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_15-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_15-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_77KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_77KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_77KB-3-of-4.tfrecord\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start generate TD_77KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_64-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_64-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_64-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_64-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_80KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_80KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_80KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_96KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_80KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_96KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_66-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_96KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_66-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_96KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_66-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_66-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_73KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_73KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_95KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_73KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_95KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_20-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_73KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_95KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_20-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_95KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_20-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_75KB-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_20-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_75KB-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_75KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_71KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_47-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_75KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_71KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_44-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_54-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_47-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_71KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_44-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_54-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_59-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_47-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_71KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_44-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_54-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_59-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_47-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_44-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_54-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_25-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_59-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_52-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_25-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_59-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_52-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_25-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_52-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_25-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_52-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_36-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_2-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_36-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_2-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_36-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_2-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_36-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_2-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_40-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_93KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_40-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_93KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_40-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_93KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_40-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_93KB-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_9-1-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_9-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_9-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_9-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_78KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_88KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_76KB-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_53-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_78KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_88KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_76KB-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_53-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_78KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_88KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_76KB-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_53-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_78KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_88KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_48-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_76KB-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_58-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_60-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_53-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_48-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_58-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_60-2-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_29-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_5-1-of-4.tfrecord\n",
      "\n",
      "Start generate TD_48-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_58-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_60-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_29-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_5-2-of-4.tfrecord\n",
      "\n",
      "Start generate TD_48-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_58-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_60-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_29-3-of-4.tfrecord\n",
      "\n",
      "Start generate TD_5-3-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "\n",
      "Start generate TD_29-4-of-4.tfrecord\n",
      "\n",
      "Start generate TD_5-4-of-4.tfrecord\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n",
      "counter reach 2000\n",
      "Transform done!\n",
      "total 2000 data processed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process ForkPoolWorker-31:\n",
      "Process ForkPoolWorker-25:\n",
      "Process ForkPoolWorker-13:\n",
      "Process ForkPoolWorker-20:\n",
      "Process ForkPoolWorker-30:\n",
      "Process ForkPoolWorker-16:\n",
      "Process ForkPoolWorker-11:\n",
      "Process ForkPoolWorker-33:\n",
      "Process ForkPoolWorker-34:\n",
      "Process ForkPoolWorker-27:\n",
      "Process ForkPoolWorker-21:\n",
      "Process ForkPoolWorker-23:\n",
      "Process ForkPoolWorker-12:\n",
      "Process ForkPoolWorker-15:\n",
      "Process ForkPoolWorker-29:\n",
      "Process ForkPoolWorker-18:\n",
      "Process ForkPoolWorker-22:\n",
      "Process ForkPoolWorker-28:\n",
      "Process ForkPoolWorker-32:\n",
      "Process ForkPoolWorker-14:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n"
     ]
    }
   ],
   "source": [
    "%run h5_to_tfrecord.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34eb5066",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %run h5_to_tfrecord.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53fba57a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def _collection(dataset_path, speech_txt, idx, target_sr = 48000):\n",
    "#     speech_list = open(speech_txt).readlines() \n",
    "#     num_of_speech = len(speech_list) \n",
    "# ######################################################################\n",
    "# ##                         Speech selection                         ## \n",
    "# ######################################################################\n",
    "#     speech_idx = idx\n",
    "#     speech_filename = os.path.join(dataset_path, speech_list[speech_idx].rstrip('\\n'))\n",
    "    \n",
    "#     y, sr = load(speech_filename, sr=None)\n",
    "#     sr, y = sr_check(y, sr)\n",
    "#     y = type_check(y)\n",
    "\n",
    "#     return y, idx+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de377f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def generator(file_idx=0, idx = 0):\n",
    "#     while True:\n",
    "#         h5f_noisy_td = h5py.File(mother_path + '/TD_' + str(file_idx) + '_noise.h5', 'a')\n",
    "        \n",
    "#         noise, idx = _collection(dataset_path= '/home3/user/myhsueh/',\n",
    "#                                  speech_txt= '/home3/user/myhsueh/training_set_noise.txt', \n",
    "#                                  idx = idx)\n",
    "        \n",
    "#         if not noise.shape[0] % length == 0 :\n",
    "#             start_pad = random.randint(0,length - noise.shape[0] % length)\n",
    "#             end_pad = length - noise.shape[0] % length - start_pad\n",
    "#             noise = np.pad(noise,(start_pad,end_pad))\n",
    "            \n",
    "#         data_noise_td = np.empty((noise.shape[0]//length, length))\n",
    "\n",
    "#         for j in range(noise.shape[0]//length):\n",
    "#             data_noise_td[j] = noise[j*length:(j+1)*length]\n",
    "        \n",
    "#         idx_add = save_h5(h5f_noisy_td,np.array(data_noise_td),'data',max_len=2000, flag=True)\n",
    "        \n",
    "#         h5f_noisy_td.close()\n",
    "        \n",
    "#         if idx_add == True: break\n",
    "            \n",
    "#     return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c0e7981",
   "metadata": {},
   "outputs": [],
   "source": [
    "# idx = 0\n",
    "# for i in range(50):\n",
    "#     idx = generator(i, idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fd32236",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f2dfaea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import h5py\n",
    "# import numpy as np\n",
    "\n",
    "# for i in range(99, 49, -1):\n",
    "#     filename = '/home3/user/myhsueh/h5_dataset/TD_' + str(i) + '_clean.h5'\n",
    "#     filename2 = '/home3/user/myhsueh/h5_dataset/TD_' + str(i) + '_noisy.h5'\n",
    "#     with h5py.File(filename, 'r') as hf:\n",
    "#         data = hf['data'][:]\n",
    "#     with h5py.File(filename2, 'r') as hf:\n",
    "#         data2 = hf['data'][:]\n",
    "        \n",
    "#     nan_check = np.sum(np.isnan(data)) + np.sum(np.isnan(data2))\n",
    "#     inf_check = np.sum(np.isinf(data)) + np.sum(np.isinf(data2))\n",
    "#     if nan_check>0 or inf_check>0:\n",
    "#         print('i:%d, nan:%d, inf:%d'%(i, nan_check, inf_check))\n",
    "#     else:\n",
    "#         print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72f779b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filename = '/home3/user/myhsueh/h5_dataset/TD_0_clean.h5'\n",
    "# with h5py.File(filename, 'r') as hf:\n",
    "#     data = hf['data'][:]\n",
    "\n",
    "# nan_check = np.sum(numpy.isnan(data))\n",
    "# inf_check = np.sum(numpy.isinf(data))\n",
    "# print('i:%d, nan:%d, inf:%d'%(i, nan_check, inf_check))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c701e8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
