SLTA-a.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from six.moves import xrange\n",
    "from sklearn.metrics import average_precision_score\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import pickle\n",
    "import operator\n",
    "import os\n",
    "import io\n",
    "import random\n",
    "import re\n",
    "from tensorflow.python.framework import dtypes\n",
    "from tensorflow.python.ops.nn import dropout as drop\n",
    "#from tensorflow.python.ops import rnn, rnn_cell\n",
    "from tensorflow.contrib import rnn\n",
    "from math import sqrt\n",
    "import random\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#keras\n",
    "import heapq\n",
    "import random\n",
    "from tqdm import tqdm,trange\n",
    "import pandas as pd\n",
    "import gensim\n",
    "import keras.backend.tensorflow_backend as KTF\n",
    "import keras.backend as K\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten,concatenate,multiply,RepeatVector,Dot,Activation,Reshape\n",
    "from keras.layers import Input, LSTM, Embedding, Dense,TimeDistributed,CuDNNGRU,CuDNNLSTM,GRU,LSTM,Lambda,Dropout,Bidirectional\n",
    "from keras.regularizers import l2\n",
    "from keras.models import Model, Sequential\n",
    "from keras.preprocessing import image\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.callbacks import EarlyStopping,Callback\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from keras.optimizers import Adam,Adadelta\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.utils import multi_gpu_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#shape=(length, 256)，tf.float = 32\n",
    "def htanh(input_tensor,units=256):\n",
    "    tanh = Dense(units,activation='tanh')(input_tensor)\n",
    "    sigmoid = Dense(units,activation='sigmoid')(input_tensor)\n",
    "    return multiply([tanh,sigmoid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#top_down_attention\n",
    "def top_down_attention(image_vec,encoded_vec):\n",
    "    K_size,image_dim = image_vec.shape[1].value,image_vec.shape[-1].value\n",
    "    repet_vec = RepeatVector(K_size)(encoded_vec)\n",
    "    concat_vec = concatenate([image_vec,repet_vec])\n",
    "    attention_temp = htanh(concat_vec,512)\n",
    "    attention_a = Dense(1, activation=None)(attention_temp)\n",
    "    attention_weight = Activation(activation='softmax')(attention_a)\n",
    "    attention_output = Dot(axes=1)([attention_weight, image_vec])\n",
    "    attention_output = Reshape([image_dim])(attention_output)\n",
    "    return attention_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_IoU(i0, i1):\n",
    "    union = (min(i0[0], i1[0]), max(i0[1], i1[1]))\n",
    "    inter = (max(i0[0], i1[0]), min(i0[1], i1[1]))\n",
    "    iou = 1.0*(inter[1]-inter[0])/(union[1]-union[0])\n",
    "    return iou\n",
    "\n",
    "def calculate_nIoL(base, sliding_clip):\n",
    "    inter = (max(base[0], sliding_clip[0]), min(base[1], sliding_clip[1]))\n",
    "    inter_l = inter[1]-inter[0]\n",
    "    length = sliding_clip[1]-sliding_clip[0]\n",
    "    nIoL = 1.0*(length-inter_l)/length\n",
    "    return nIoL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#word embedding\n",
    "PAD_IDENTIFIER = '<pad>'\n",
    "UNK_IDENTIFIER = '<unk>' # <unk> is the word used to identify unknown words\n",
    "SENTENCE_SPLIT_REGEX = re.compile(r'(\\W+)')\n",
    "wordembed_params = './word_embedding/embed_matrix.npy'\n",
    "embedding_mat = np.load(wordembed_params)\n",
    "vocab_file = './word_embedding/vocabulary_72700.txt'\n",
    "T=10#MAX_WORDS = 10\n",
    "\n",
    "def load_vocab_dict_from_file(dict_file, pad_at_first=True):\n",
    "            with io.open(dict_file, encoding='utf-8') as f:\n",
    "                words = [w.strip() for w in f.readlines()]\n",
    "            if pad_at_first and words[0] != '<pad>':\n",
    "                raise Exception(\"The first word needs to be <pad> in the word list.\")\n",
    "            vocab_dict = {words[n]: n for n in range(len(words))}\n",
    "            return vocab_dict\n",
    "\n",
    "def sentence2vocab_indices(sentence, vocab_dict):\n",
    "            if isinstance(sentence, bytes):\n",
    "                sentence = sentence.decode()\n",
    "            words = SENTENCE_SPLIT_REGEX.split(sentence.strip())\n",
    "            words = [w.lower() for w in words if len(w.strip()) > 0]\n",
    "            # remove .\n",
    "            if len(words) > 0 and (words[-1] == '.' or words[-1] == '?'):\n",
    "                words = words[:-1]\n",
    "            vocab_indices = [(vocab_dict[w] if w in vocab_dict else vocab_dict[UNK_IDENTIFIER])\n",
    "                             for w in words]\n",
    "            return vocab_indices\n",
    "\n",
    "def preprocess_vocab_indices(vocab_indices, vocab_dict, T):\n",
    "            # Truncate long sentences\n",
    "            if len(vocab_indices) > T:\n",
    "                vocab_indices = vocab_indices[:T]\n",
    "            # Pad short sentences at the beginning with the special symbol '<pad>' \n",
    "            if len(vocab_indices) < T:\n",
    "                vocab_indices = [vocab_dict[PAD_IDENTIFIER]] * (T - len(vocab_indices)) + vocab_indices\n",
    "            return vocab_indices\n",
    "\n",
    "def preprocess_sentence(sentence, vocab_dict, T):\n",
    "            vocab_indices = sentence2vocab_indices(sentence, vocab_dict)\n",
    "            return preprocess_vocab_indices(vocab_indices, vocab_dict, T)\n",
    "\n",
    "vocab_dict = load_vocab_dict_from_file(vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainingDataSet(object):\n",
    "    def __init__(self, it_path, batch_size,sliding_dir,train_fast_rcnn_path):\n",
    "\n",
    "        self.batch_size = batch_size\n",
    "        self.context_size = 128\n",
    "        self.context_num = 1\n",
    "        self.visual_feature_dim=4096\n",
    "        self.clip_sentence_pairs_iou=pickle.load(open(it_path,'rb'))\n",
    "        self.sliding_clip_path=sliding_dir\n",
    "        #train_fast_rcnn_path\n",
    "        self.train_fast_rcnn_path = train_fast_rcnn_path\n",
    "        \n",
    "        self.num_samples_iou=len(self.clip_sentence_pairs_iou)\n",
    "        print (str(len(self.clip_sentence_pairs_iou))+\" iou clip-sentence pairs are readed\")\n",
    "\n",
    "        \n",
    "    def get_context_window(self, clip_name, win_length):\n",
    "        movie_name = clip_name.split(\"_\")[0]\n",
    "        start = int(clip_name.split(\"_\")[1])\n",
    "        end = int(clip_name.split(\"_\")[2].split(\".\")[0])\n",
    "        clip_length = self.context_size\n",
    "        left_context_feats = np.zeros([win_length, 4096], dtype=np.float32)\n",
    "        right_context_feats = np.zeros([win_length, 4096], dtype=np.float32)\n",
    "        last_left_feat = np.load(self.sliding_clip_path+clip_name)\n",
    "        last_right_feat = np.load(self.sliding_clip_path+clip_name)\n",
    "        for k in range(win_length):\n",
    "            left_context_start = start-clip_length*(k+1)\n",
    "            left_context_end = start-clip_length*k\n",
    "            right_context_start = end+clip_length*k\n",
    "            right_context_end = end+clip_length*(k+1)\n",
    "            left_context_name = movie_name+\"_\"+str(left_context_start)+\"_\"+str(left_context_end)+\".npy\"\n",
    "            right_context_name = movie_name+\"_\"+str(right_context_start)+\"_\"+str(right_context_end)+\".npy\"\n",
    "            if os.path.exists(self.sliding_clip_path+left_context_name):\n",
    "                left_context_feat = np.load(self.sliding_clip_path+left_context_name)\n",
    "                last_left_feat = left_context_feat\n",
    "            else:\n",
    "                left_context_feat = last_left_feat\n",
    "            if os.path.exists(self.sliding_clip_path+right_context_name):\n",
    "                right_context_feat = np.load(self.sliding_clip_path+right_context_name)\n",
    "                last_right_feat = right_context_feat\n",
    "            else:\n",
    "                right_context_feat = last_right_feat\n",
    "            left_context_feats[k] = left_context_feat\n",
    "            right_context_feats[k] = right_context_feat\n",
    "        return left_context_feats, right_context_feats\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    def get_fast_rcnn_vector(self, clip_name):\n",
    "        movie_name = clip_name.split(\"_\")[0].strip('.avi')\n",
    "        start = int(clip_name.split(\"_\")[1])\n",
    "        end = int(clip_name.split(\"_\")[2].strip('.npy'))\n",
    "        vec_length = end-start\n",
    "        name = self.train_fast_rcnn_path+movie_name+'.pkl'\n",
    "        vecs = pickle.load(open(name,'rb'))\n",
    "        seq_len = 64\n",
    "        if vec_length < seq_len:\n",
    "            clip_vec = vecs[start-1:end-1]\n",
    "            clip_vec = np.vstack((clip_vec,np.zeros([seq_len-vec_length,2048])))\n",
    "        else:\n",
    "            clip_vec = vecs[start-1:start+63]\n",
    "        return clip_vec\n",
    "   \n",
    "    \n",
    "\n",
    "\n",
    "    def next_batch_iou(self):\n",
    "        \n",
    "        random_batch_index = random.sample(range(self.num_samples_iou), self.batch_size)\n",
    "        #fast_rcnn_vec_batch\n",
    "        fast_rcnn_vec_batch = np.zeros([self.batch_size,64,2048])\n",
    "        image_batch = np.zeros([self.batch_size, self.visual_feature_dim, 2 * self.context_num + 1])\n",
    "        text_seq_batch = []\n",
    "        noun_seq_batch = []\n",
    "        \n",
    "        offset_batch = np.zeros([self.batch_size, 2], dtype=np.float32)\n",
    "        index = 0\n",
    "        clip_set = set()\n",
    "        while index < self.batch_size:\n",
    "            k = random_batch_index[index]\n",
    "            clip_name = self.clip_sentence_pairs_iou[k][0]\n",
    "                \n",
    "            if not clip_name in clip_set:\n",
    "                clip_set.add(clip_name)\n",
    "                movie_name=clip_name.split('_')[0]\n",
    "                \n",
    "                \n",
    "                clip_vec = self.get_fast_rcnn_vector(self.clip_sentence_pairs_iou[k][2])#(64,2048)\n",
    "                fast_rcnn_vec_batch[index, :, :] = clip_vec\n",
    "               \n",
    "                noun_seq_batch.append(preprocess_sentence(self.clip_sentence_pairs_iou[k][5], vocab_dict, T))\n",
    "                       \n",
    "             \n",
    "                feat_path = self.sliding_clip_path+self.clip_sentence_pairs_iou[k][2]\n",
    "                featmap = np.load(feat_path)\n",
    "                # read context features\n",
    "                left_context_feat, right_context_feat = self.get_context_window(self.clip_sentence_pairs_iou[k][2], self.context_num)\n",
    "                left_context_feat = np.reshape(left_context_feat, [self.visual_feature_dim])\n",
    "                right_context_feat = np.reshape(right_context_feat, [self.visual_feature_dim])            \n",
    "                image_batch[index, :, :] = np.column_stack((left_context_feat, featmap, right_context_feat))#batchsize,4096,3       \n",
    "                text_seq_batch.append(preprocess_sentence(self.clip_sentence_pairs_iou[k][1], vocab_dict, T))\n",
    "                #iou\n",
    "                p_offset = self.clip_sentence_pairs_iou[k][3]#start_current-start_left -62\n",
    "                l_offset = self.clip_sentence_pairs_iou[k][4]#end_current-end_left    41\n",
    "                offset_batch[index, 0] = p_offset\n",
    "                offset_batch[index, 1] = l_offset\n",
    "                index += 1\n",
    "            else:\n",
    "                r = random.choice(range(self.num_samples_iou))\n",
    "                random_batch_index[index] = r\n",
    "                continue\n",
    "           \n",
    "        return image_batch, text_seq_batch, offset_batch,fast_rcnn_vec_batch,noun_seq_batch  \n",
    "        #(batchsize,4096,3) (batchsize,T) (batchsize,64,2048) (batchsize,T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestingDataSet(object):\n",
    "    def __init__(self, csv_path, batch_size,img_dir,test_fast_rcnn_path):\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.image_dir = img_dir\n",
    "        self.visual_feature_dim=4096\n",
    "        print (\"Reading testing data list from \")\n",
    "        self.sliding_clip_path = img_dir\n",
    "        self.test_fast_rcnn_path = test_fast_rcnn_path\n",
    "        \n",
    "        self.clip_sentence_pairs=pickle.load(open(csv_path,'rb'))\n",
    "       \n",
    "        movie_names_set = set()\n",
    "        self.movie_clip_names = {}\n",
    "        for k in range(len(self.clip_sentence_pairs)):\n",
    "            clip_name = self.clip_sentence_pairs[k][0]\n",
    "            movie_name = clip_name.split(\"_\")[0]\n",
    "            if not movie_name in movie_names_set:\n",
    "                movie_names_set.add(movie_name)\n",
    "                self.movie_clip_names[movie_name] = []\n",
    "            self.movie_clip_names[movie_name].append(k)\n",
    "        self.movie_names = list(movie_names_set)\n",
    "\n",
    "        self.clip_num_per_movie_max = 0\n",
    "        for movie_name in self.movie_clip_names:\n",
    "            if len(self.movie_clip_names[movie_name])>self.clip_num_per_movie_max: self.clip_num_per_movie_max = len(self.movie_clip_names[movie_name])\n",
    "        print (\"Max number of clips in a movie is \"+str(self.clip_num_per_movie_max))\n",
    "\n",
    "        \n",
    "        \n",
    "\n",
    "    def get_fast_rcnn_vector(self, clip_name):\n",
    "        movie_name = clip_name.split(\"_\")[0].strip('.avi')\n",
    "        start = int(clip_name.split(\"_\")[1])\n",
    "        end = int(clip_name.split(\"_\")[2].strip('.npy'))\n",
    "        vec_length = end-start\n",
    "        name = self.test_fast_rcnn_path+movie_name+'.pkl'\n",
    "        vecs = pickle.load(open(name,'rb'))\n",
    "        seq_len = 64\n",
    "        if vec_length < seq_len:\n",
    "            clip_vec = vecs[start-1:end-1]\n",
    "            clip_vec = np.vstack((clip_vec,np.zeros([seq_len-vec_length,2048])))\n",
    "        else:\n",
    "            clip_vec = vecs[start-1:start+63]\n",
    "        return clip_vec\n",
    "             \n",
    "        \n",
    "    def get_context_window(self, clip_name, win_length):\n",
    "        movie_name = clip_name.split(\"_\")[0]\n",
    "        start = int(clip_name.split(\"_\")[1])\n",
    "        end = int(clip_name.split(\"_\")[2].split('.')[0])\n",
    "        clip_length = 128\n",
    "        left_context_feats = np.zeros([win_length,4096], dtype=np.float32)\n",
    "        right_context_feats = np.zeros([win_length,4096], dtype=np.float32)\n",
    "        last_left_feat = np.load(self.sliding_clip_path+clip_name)\n",
    "        last_right_feat = np.load(self.sliding_clip_path+clip_name)\n",
    "        for k in range(win_length):\n",
    "            left_context_start = start-clip_length*(k+1)\n",
    "            left_context_end = start-clip_length*k\n",
    "            right_context_start = end+clip_length*k\n",
    "            right_context_end = end+clip_length*(k+1)\n",
    "            left_context_name = movie_name+\"_\"+str(left_context_start)+\"_\"+str(left_context_end)+\".npy\"\n",
    "            right_context_name = movie_name+\"_\"+str(right_context_start)+\"_\"+str(right_context_end)+\".npy\"\n",
    "            if os.path.exists(self.sliding_clip_path+left_context_name):\n",
    "                left_context_feat = np.load(self.sliding_clip_path+left_context_name)\n",
    "                last_left_feat = left_context_feat\n",
    "            else:\n",
    "                left_context_feat = last_left_feat\n",
    "            if os.path.exists(self.sliding_clip_path+right_context_name):\n",
    "                right_context_feat = np.load(self.sliding_clip_path+right_context_name)\n",
    "                last_right_feat = right_context_feat\n",
    "            else:\n",
    "                right_context_feat = last_right_feat\n",
    "            left_context_feats[k] = left_context_feat\n",
    "            right_context_feats[k] = right_context_feat\n",
    "\n",
    "        return left_context_feats,right_context_feats\n",
    "\n",
    "        \n",
    "        \n",
    "    def load_movie_slidingclip(self, movie_name, sample_num):\n",
    "        movie_clip_sentences = []\n",
    "        movie_clip_featmap = []\n",
    "\n",
    "        for k in range(len(self.clip_sentence_pairs)):\n",
    "            if movie_name in self.clip_sentence_pairs[k][0]:\n",
    "                #clip_name,query,noun_word\n",
    "                movie_clip_sentences.append((self.clip_sentence_pairs[k][0], self.clip_sentence_pairs[k][1],self.clip_sentence_pairs[k][2]))\n",
    "        sliding_clip_names=os.listdir(self.sliding_clip_path)\n",
    "        for k in range(len(sliding_clip_names)):\n",
    "           \n",
    "            if \"npy\" in sliding_clip_names[k]:\n",
    "                \n",
    "                if movie_name in sliding_clip_names[k]:\n",
    "                    # print (str(k)+\"/\"+str(len(self.movie_clip_names[movie_name])))\n",
    "                    visual_feature_path = self.sliding_clip_path+sliding_clip_names[k]#clip_name\n",
    "                    #context_feat=self.get_context(self.sliding_clip_names[k]+\".npy\")\n",
    "                    left_context_feat,right_context_feat = self.get_context_window(sliding_clip_names[k],1)\n",
    "                    feature_data = np.load(visual_feature_path)\n",
    "                    left_context_feat=np.reshape(left_context_feat,[self.visual_feature_dim])\n",
    "                    right_context_feat=np.reshape(right_context_feat,[self.visual_feature_dim])\n",
    "                    \n",
    "                    #fast_rcnn_vector\n",
    "                    fast_rcnn_vec = self.get_fast_rcnn_vector(sliding_clip_names[k])#64*2048\n",
    "                    \n",
    "\n",
    "                    #comb_feat=np.hstack((context_feat,feature_data))\n",
    "                    comb_feat = np.column_stack((left_context_feat,feature_data,right_context_feat))\n",
    "                    movie_clip_featmap.append((sliding_clip_names[k], comb_feat,fast_rcnn_vec))\n",
    "        return movie_clip_featmap, movie_clip_sentences\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_layer(name, bottom, kernel_size, stride, output_dim, padding='SAME',\n",
    "               bias_term=True, weights_initializer=None, biases_initializer=None):\n",
    "    # input has shape [batch, in_height, in_width, in_channels]\n",
    "    input_dim = bottom.get_shape().as_list()[-1]\n",
    "\n",
    "    with tf.variable_scope(name):\n",
    "        if weights_initializer is None and biases_initializer is None:\n",
    "            # initialize the variables\n",
    "            if weights_initializer is None:\n",
    "                weights_initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)\n",
    "            if bias_term and biases_initializer is None:\n",
    "                biases_initializer = tf.constant_initializer(0.)\n",
    "            print (\"input_dim\"+str(input_dim))\n",
    "            # filter has shape [filter_height, filter_width, in_channels, out_channels]\n",
    "            weights = tf.get_variable(\"weights\",\n",
    "                [kernel_size, kernel_size, input_dim, output_dim],\n",
    "                initializer=weights_initializer)\n",
    "            if bias_term:\n",
    "                biases = tf.get_variable(\"biases\", output_dim,\n",
    "                    initializer=biases_initializer)\n",
    "            print (str(weights.name)+\" initialized as random or retrieved from graph\")\n",
    "            if bias_term:\n",
    "                print (biases.name+\" initialized as random or retrieved from graph\")\n",
    "\n",
    "        else:\n",
    "            weights = tf.get_variable(\"weights\",\n",
    "                shape=None,\n",
    "                initializer=weights_initializer)\n",
    "            if bias_term:\n",
    "                biases = tf.get_variable(\"biases\", shape=None,\n",
    "                    initializer=biases_initializer) \n",
    "\n",
    "            print (weights.name+\" initialized from pre-trained parameters or retrieved from graph\")\n",
    "            if bias_term:\n",
    "                print (biases.name+\" initialized from pre-trained parameters or retrieved from graph\")\n",
    "\n",
    "\n",
    "    conv = tf.nn.conv2d(bottom, filter=weights,\n",
    "        strides=[1, stride, stride, 1], padding=padding)\n",
    "    if bias_term:\n",
    "        conv = tf.nn.bias_add(conv, biases)\n",
    "    return conv\n",
    "\n",
    "def conv_relu_layer(name, bottom, kernel_size, stride, output_dim, padding='SAME',\n",
    "                    bias_term=True, weights_initializer=None, biases_initializer=None):\n",
    "    conv = conv_layer(name, bottom, kernel_size, stride, output_dim, padding,\n",
    "                      bias_term, weights_initializer, biases_initializer)\n",
    "    relu = tf.nn.relu(conv)\n",
    "    return relu\n",
    "\n",
    "def deconv_layer(name, bottom, kernel_size, stride, output_dim, padding='SAME',\n",
    "                 bias_term=True, weights_initializer=None, biases_initializer=None):\n",
    "    # input_shape is [batch, in_height, in_width, in_channels]\n",
    "    input_shape = bottom.get_shape().as_list()\n",
    "    batch_size, input_height, input_width, input_dim = input_shape\n",
    "    output_shape = [batch_size, input_height*stride, input_width*stride, output_dim]\n",
    "\n",
    "    # weights and biases variables\n",
    "    with tf.variable_scope(name):\n",
    "        # initialize the variables\n",
    "        if weights_initializer is None:\n",
    "            weights_initializer = tf.random_normal_initializer()\n",
    "        if bias_term and biases_initializer is None:\n",
    "            biases_initializer = tf.constant_initializer(0.)\n",
    "\n",
    "        # filter has shape [filter_height, filter_width, out_channels, in_channels]\n",
    "        weights = tf.get_variable(\"weights\",\n",
    "            [kernel_size, kernel_size, output_dim, input_dim],\n",
    "            initializer=weights_initializer)\n",
    "        if bias_term:\n",
    "            biases = tf.get_variable(\"biases\", output_dim,\n",
    "                initializer=biases_initializer)\n",
    "\n",
    "    deconv = tf.nn.conv2d_transpose(bottom, filter=weights,\n",
    "        output_shape=output_shape, strides=[1, stride, stride, 1],\n",
    "        padding=padding)\n",
    "    if bias_term:\n",
    "        deconv = tf.nn.bias_add(deconv, biases)\n",
    "    return deconv\n",
    "\n",
    "\n",
    "def deconv_relu_layer(name, bottom, kernel_size, stride, output_dim, padding='SAME',\n",
    "                      bias_term=True, weights_initializer=None, biases_initializer=None):\n",
    "    deconv = deconv_layer(name, bottom, kernel_size, stride, output_dim, padding,\n",
    "                          bias_term, weights_initializer, biases_initializer)\n",
    "    relu = tf.nn.relu(deconv)\n",
    "    return relu\n",
    "\n",
    "def pooling_layer(name, bottom, kernel_size, stride):\n",
    "    pool = tf.nn.max_pool(bottom, ksize=[1, kernel_size, kernel_size, 1],\n",
    "        strides=[1, stride, stride, 1], padding='SAME', name=name)\n",
    "    return pool\n",
    "\n",
    "def fc(name, bottom, output_dim, bias_term=True, weights_initializer=None,\n",
    "             biases_initializer=None):\n",
    "\n",
    "    shape = bottom.get_shape().as_list()\n",
    "    input_dim = 1\n",
    "\n",
    "    for d in shape[1:]:\n",
    "        input_dim *= d\n",
    "    flat_bottom = tf.reshape(bottom, [-1, input_dim])\n",
    "    \n",
    "    # weights and biases variables\n",
    "    with tf.variable_scope(name):\n",
    "        if weights_initializer is None and biases_initializer is None:\n",
    "            # initialize the variables\n",
    "            if weights_initializer is None:\n",
    "                weights_initializer = tf.random_normal_initializer()\n",
    "            if bias_term and biases_initializer is None:\n",
    "                biases_initializer = tf.constant_initializer(0.)\n",
    "\n",
    "            # weights has shape [input_dim, output_dim]\n",
    "            weights = tf.get_variable(\"weights\", [input_dim, output_dim],\n",
    "                initializer=weights_initializer)\n",
    "            if bias_term:\n",
    "                biases = tf.get_variable(\"biases\", output_dim,\n",
    "                    initializer=biases_initializer)\n",
    "\n",
    "            print (weights.name+\" initialized as random or retrieved from graph\")\n",
    "            if bias_term:\n",
    "                print (biases.name+\" initialized as random or retrieved from graph\")\n",
    "        else:\n",
    "            weights = tf.get_variable(\"weights\", shape=None,\n",
    "                initializer=weights_initializer)\n",
    "            if bias_term:\n",
    "                biases = tf.get_variable(\"biases\", shape=None,\n",
    "                    initializer=biases_initializer)\n",
    "\n",
    "            print (weights.name+\" initialized from pre-trained parameters or retrieved from graph\")\n",
    "            if bias_term:\n",
    "                print (biases.name+\" initialized from pre-trained parameters or retrieved from graph\")\n",
    "\n",
    "    if bias_term:\n",
    "        fc = tf.nn.xw_plus_b(flat_bottom, weights, biases)\n",
    "    else:\n",
    "        fc = tf.matmul(flat_bottom, weights)\n",
    "    return fc\n",
    "\n",
    "def fc_relu_layer(name, bottom, output_dim, bias_term=True,\n",
    "                  weights_initializer=None, biases_initializer=None):\n",
    "    fc = fc(name, bottom, output_dim, bias_term, weights_initializer,\n",
    "                  biases_initializer)\n",
    "    relu = tf.nn.relu(fc)\n",
    "    return relu\n",
    "\n",
    "def softmax_loss_layer(name, score_bottom, label_bottom):\n",
    "    # Check shape\n",
    "    score_shape = score_bottom.get_shape().as_list()\n",
    "    label_shape = label_bottom.get_shape().as_list()\n",
    "    assert len(score_shape) == len(label_shape) + 1\n",
    "    assert score_shape[:-1] == label_shape\n",
    "\n",
    "    # Compute the outer dimensions dimensions in label\n",
    "    inner_dim = score_shape[-1]\n",
    "    outer_dim = 1\n",
    "    for d in label_shape: outer_dim *= d\n",
    "\n",
    "    # flatten score and label\n",
    "    flat_score = tf.reshape(score_bottom, [outer_dim, inner_dim])\n",
    "    flat_label = tf.reshape(label_bottom, [outer_dim, 1])\n",
    "\n",
    "    # Reshape the labels into a dense Tensor of\n",
    "    # shape [batch_size, NUM_CLASSES].\n",
    "    sparse_labels = tf.reshape(labels, [FLAGS.batch_size, 1])\n",
    "    indices = tf.reshape(tf.range(FLAGS.batch_size), [FLAGS.batch_size, 1])\n",
    "    concated = tf.concat([indices, sparse_labels],1)\n",
    "    dense_labels = tf.sparse_to_dense(concated, [FLAGS.batch_size, NUM_CLASSES],\n",
    "        1.0, 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vs_multilayer(input_batch,name,middle_layer_dim=1000,reuse=False):\n",
    "    with tf.variable_scope(name):\n",
    "        if reuse==True:\n",
    "            print (name+\" reuse variables\")\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "        else:\n",
    "            print (name+\" doesn't reuse variables\")\n",
    "\n",
    "        layer1 = conv_relu_layer('layer1', input_batch,\n",
    "                        kernel_size=1,stride=1,output_dim=middle_layer_dim)\n",
    "        sim_score = conv_layer('layer2', layer1,\n",
    "                        kernel_size=1,stride=1,output_dim=3)\n",
    "    return sim_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordembed_params = './word_embedding/embed_matrix.npy'\n",
    "embedding_mat = np.load(wordembed_params)#(72704, 300)\n",
    "class SLAT_Model(object):\n",
    "    #T=10,train_feature_dir,test_feature_dir,semantic_size=1024,mpl_hidden=1000)\n",
    "    def __init__(self, batch_size, train_visual_feature_dir, test_visual_feature_dir, lr, n_input_visual, n_input_text,\n",
    "                   n_hidden_text, n_step_text,train_sliding_dir,test_sliding_dir,semantic_size,mpl_hidden,train_fast_rcnn_path,test_fast_rcnn_path):\n",
    "        self.batch_size = batch_size#30\n",
    "        self.test_batch_size = 1\n",
    "        self.vs_lr = lr#0.01\n",
    "        self.n_input_visual = n_input_visual #4096\n",
    "        self.n_input_text = n_input_text#300\n",
    "        self.n_step_text = n_step_text#T=10\n",
    "        self.n_hidden_text=n_hidden_text\n",
    "        self.context_size = 128\n",
    "        self.context_num = 1\n",
    "        self.visual_feature_dim=4096\n",
    "        self.lambda_regression = 0.01#self.lambda_regression*loss_reg+loss_aln\n",
    "        self.alpha = 1.0 / batch_size\n",
    "        self.train_set=TrainingDataSet(train_visual_feature_dir, self.batch_size,train_sliding_dir,train_fast_rcnn_path) \n",
    "        self.test_set=TestingDataSet(test_visual_feature_dir, self.test_batch_size,test_sliding_dir,test_fast_rcnn_path)\n",
    "        self.semantic=semantic_size#1024\n",
    "        self.mpl_hidden=mpl_hidden#1000\n",
    "    '''\n",
    "    used in training alignment model, ROLE(aln)\n",
    "    '''\n",
    "    def fill_feed_dict_train(self):\n",
    "        image_batch,sentence_batch,offset_batch,fast_rcnn_vec_batch,noun_seq_batch  = self.train_set.next_batch()\n",
    "        input_feed = {\n",
    "                self.visual_featmap_ph_train: image_batch,#(batchsize,4096,3)\n",
    "                self.sentence_ph_train: sentence_batch,#(batchsize,T)\n",
    "                self.offset_ph: offset_batch,#(batchsize,2)\n",
    "            \n",
    "                self.fast_rcnn_vec_train: fast_rcnn_vec_batch,#(batchsize,64,2048)\n",
    "                self.noun_word_ph_train: noun_seq_batch#(batchsize,T)\n",
    "                       \n",
    "            \n",
    "        }\n",
    "\n",
    "        return input_feed\n",
    "    \n",
    "    '''\n",
    "    used in training alignment+regression model, ROLE(reg)\n",
    "    \n",
    "    '''\n",
    "    def fill_feed_dict_train_reg(self):\n",
    "        image_batch, sentence_batch, offset_batch,fast_rcnn_vec_batch,noun_seq_batch = self.train_set.next_batch_iou()\n",
    "        input_feed = {\n",
    "                self.visual_featmap_ph_train: image_batch,  #batch_size,4096,3\n",
    "                self.sentence_ph_train: sentence_batch,  #batch_size*(10)\n",
    "                self.offset_ph: offset_batch,   #(batchsize,2)\n",
    "                self.fast_rcnn_vec_train: fast_rcnn_vec_batch,  #(batchsize,64,2048)\n",
    "                self.noun_word_ph_train: noun_seq_batch    #(batchsize,T)        \n",
    "        }\n",
    "\n",
    "        return input_feed\n",
    "\n",
    "    def bilstm(self,x):\n",
    "        \"\"\"RNN (LSTM or GRU) model for image\"\"\"\n",
    "        # x.shape [N,T,D] batch_size，10，300\n",
    "        x=tf.transpose(x,[1,0,2])# [T,N,D] 10，batch_size,300\n",
    "        fw_x = tf.reshape(x, [-1, self.n_input_text]) # 10*batch,300\n",
    "        fw_x = tf.split(fw_x, self.n_step_text,0)\n",
    "        with tf.variable_scope('bilstm_lt'):\n",
    "            #one-layer bilstm,n_hidden_text=1000\n",
    "            lstm_fw_cell = rnn.BasicLSTMCell(self.n_hidden_text, forget_bias=1.0, state_is_tuple=True)\n",
    "            lstm_bw_cell = rnn.BasicLSTMCell(self.n_hidden_text, forget_bias=1.0, state_is_tuple=True)\n",
    "            #dropout\n",
    "            #lstm_fw_cell = rnn_cell.DropoutWrapper(cell=lstm_fw_cell, input_keep_prob=1.0, output_keep_prob=keep_prob)\n",
    "            #lstm_bw_cell = rnn_cell.DropoutWrapper(cell=lstm_bw_cell, input_keep_prob=1.0, output_keep_prob=keep_prob)\n",
    "            with tf.variable_scope('fw_lt'):\n",
    "                (output_fw, state_fw) = rnn.static_rnn(lstm_fw_cell,fw_x,dtype=tf.float32)\n",
    "                t=tf.convert_to_tensor(output_fw)\n",
    "                print (t.get_shape().as_list())#10,30,1000\n",
    "            # backward direction\n",
    "            with tf.variable_scope('bw_lt'):\n",
    "                bw_x = tf.reverse(x, [0])# reverse time dim\n",
    "                bw_x = tf.reshape(bw_x, [-1, self.n_input_text])  # step*batch, feature\n",
    "                bw_x = tf.split(bw_x, self.n_step_text, 0)\n",
    "                (output_bw, state_bw) = rnn.static_rnn(lstm_bw_cell,bw_x,dtype=tf.float32)\n",
    "            # output_bw.shape = [timestep_size, batch_size, hidden_size]\n",
    "            output_bw = tf.reverse(output_bw, [0])\n",
    "            output = tf.concat([output_fw, output_bw],2)#10,30,2000    T，N，1000+1000\n",
    "        return output\n",
    "    \n",
    "    \n",
    "    def cross_modal_comb(self, input_vision_obj1,BoW_obj1,video_output_norm,batch_size):\n",
    "        #[batch_size, batch_size, 4096*3=12288]\n",
    "        vv_feature= tf.reshape(tf.tile(input_vision_obj1, [batch_size, 1]),[batch_size, batch_size, self.visual_feature_dim*(2*self.context_num+1)])\n",
    "        ss_feature= tf.reshape(tf.tile(BoW_obj1,[1, batch_size]),[batch_size, batch_size, self.n_input_text])\n",
    "        fast_rcnn_feature = tf.reshape(tf.tile(video_output_norm,[1, batch_size]),[batch_size, batch_size, 512])\n",
    "        #[1,batch_size, batch_size, 12288+300+512]\n",
    "        concat_feature1 = tf.concat([fast_rcnn_feature, ss_feature],2)\n",
    "        concat_feature = tf.reshape(tf.concat([vv_feature, concat_feature1],2),[1,batch_size, batch_size, self.visual_feature_dim*(2*self.context_num+1)+self.n_input_text+512])\n",
    "        return concat_feature\n",
    "\n",
    "    '''\n",
    "    visual semantic inference, including visual semantic alignment and clip location regression\n",
    "    '''\n",
    "    \n",
    "    \n",
    "    def visual_semantic_infer(self,visual_feature_train,text_feature_train,fast_rcnn_feature_train,noun_word_feature_train,visual_feature_test,text_feature_test,fast_rcnn_feature_test,noun_word_feature_test):\n",
    "        name=\"CTRL_Model\"\n",
    "        \n",
    "       \n",
    "        with tf.variable_scope(name):\n",
    "            \n",
    "            # text_feature_train [N,T] shape word index matrix\n",
    "            # 0. Word embedding\n",
    "            # text_seq has shape [N, T] and embedded_seq has shape [N, T, D].\n",
    "            #tex_seq [batch_size,10],embedded_seq[batch_size,10,300]\n",
    "            #embedding_mat(72704, 300)\n",
    "            embedded_seq_train = tf.nn.embedding_lookup(embedding_mat, text_feature_train)#input:N,T  output:N,T,300\n",
    "            \n",
    "            \n",
    "            # 1. Encode the sentence into a vector representation, using the final hidden states in a one-layer bidirectional LSTM network\n",
    "            q_reshape=self.bilstm(embedded_seq_train)\n",
    "            print (q_reshape.get_shape().as_list())\n",
    "            \n",
    "            \n",
    "            # 1.1 noun_word(batch,10)  embedding(batch,10,300)\n",
    "            embedded_noun_word_train = tf.nn.embedding_lookup(embedding_mat,noun_word_feature_train)\n",
    "            #GRU encode \n",
    "            #embedded_noun_word_train = tf.convert_to_tensor(embedded_noun_word_train,dtype = tf.float32)\n",
    "            encoded_vector = CuDNNGRU(512,return_state=False)(embedded_noun_word_train)#30,512\n",
    "            embedded_noun_word = encoded_vector #(30,512)\n",
    "            \n",
    "            \n",
    "            # 1.2 fast_rcnn_vec(batch,64,2048)\n",
    "            #fast_rcnn_feature_train = tf.convert_to_tensor(fast_rcnn_feature_train,dtype = tf.float32)\n",
    "            #video_vector = CuDNNGRU(512,return_state=False,return_sequences=True)(fast_rcnn_feature_train)#(batch, 64, 512)\n",
    "            video_vector = CuDNNGRU(512,return_state=False)(fast_rcnn_feature_train)#(batch,512)\n",
    "            # 1.3 top_down_attention\n",
    "            video_output = video_vector#(30,512)\n",
    "            #video_output = top_down_attention(video_vector,embedded_noun_word)#(30, 512)\n",
    "            #print (video_output.get_shape().as_list())\n",
    "\n",
    "             \n",
    "            # 2. attention units over the words in each sentence fc(fc(Q+C+PRE+POST))\n",
    "            q_reshape_flat = tf.reshape(q_reshape, [self.n_step_text * self.batch_size, self.n_hidden_text * 2])#30*10,1000*2\n",
    "            visual_feature_train=tf.transpose(visual_feature_train, [0, 2, 1])  # batch ctx fea  30,3,4096\n",
    "            visual_train=tf.reshape(visual_feature_train,[self.batch_size*(2*self.context_num+1),self.visual_feature_dim])#30*3，4096\n",
    "            query_term=fc('q2s_lt', q_reshape_flat, output_dim=self.semantic)#30*10,1024\n",
    "            moment_term=fc('c2s_lt', visual_train, output_dim=self.semantic)#30*3,1024\n",
    "\n",
    "            query_term=tf.reshape(query_term,[self.batch_size,self.n_step_text,self.semantic])#(30,10,1024)\n",
    "            moment_term=tf.reshape( moment_term,[self.batch_size,2*self.context_num+1,self.semantic])#(30,3,1024)\n",
    "            \n",
    "            term2=tf.reduce_sum(moment_term,1,keep_dims=True)\n",
    "            term2=tf.reshape(term2,[self.batch_size,self.semantic])#(30, 1024)\n",
    "            \n",
    "        \n",
    "            term2=tf.reshape(tf.tile(term2,[1,self.n_step_text]),[self.batch_size,self.n_step_text,self.semantic])\n",
    "            #(30,10,1024)add(30,10,1024),reshape(300,1024)\n",
    "            term=tf.nn.relu(tf.reshape(tf.add(query_term,term2), [self.n_step_text * self.batch_size, self.semantic]))#(300, 1024)\n",
    "            scores_obj1 = fc('fc_scores_query_lt',term, output_dim=1)\n",
    "            scores_obj1_train=tf.reshape(scores_obj1,[self.batch_size,self.n_step_text])#(30,10)\n",
    "            is_not_pad=tf.cast(tf.not_equal(text_feature_train,0),tf.float32)\n",
    "            #probs_obj1=tf.nn.softmax(scores_obj1_train)\n",
    "            #probs_obj1=tf.multiply(probs_obj1,is_not_pad)\n",
    "            probs_obj1 = tf.multiply(scores_obj1_train,is_not_pad)#(30,10)\n",
    "            probs_obj1 = probs_obj1 / tf.reduce_sum(probs_obj1, 1, keep_dims=True)#(30,10)/(30,1)\n",
    "            \n",
    "            #[N,T,embed_dim] (30,10,300)\n",
    "            temp1= tf.transpose(tf.reshape(tf.tile(probs_obj1, [1, self.n_input_text]),[self.batch_size, self.n_input_text, self.n_step_text]),[0,2,1])\n",
    "            #(30,300)\n",
    "            BoW_obj1 = tf.reduce_sum(tf.multiply(temp1,embedded_seq_train), reduction_indices=1)\n",
    "            print (BoW_obj1.get_shape().as_list())#(30,300)\n",
    "            \n",
    "            \n",
    "            \n",
    "            #3.0 visual attention part: x_i=Wv_i+b then softmax(x_i.q_j) output: [N, visual_feature_size] \n",
    "            input_vision_obj1=tf.reshape(visual_feature_train,[self.batch_size,-1])#(30,12288)\n",
    "            # cross-modal part\n",
    "            transformed_clip_train_norm = tf.nn.l2_normalize(input_vision_obj1 , dim=1)\n",
    "            transformed_obj1_sent_train_norm = tf.nn.l2_normalize(BoW_obj1, dim=1)\n",
    "            transformed_video_output_norm = tf.nn.l2_normalize(video_output, dim=1)\n",
    "            \n",
    "            \n",
    "            #(1,30,30,12588)\n",
    "            cross_modal_vec_train = self.cross_modal_comb(transformed_clip_train_norm,transformed_obj1_sent_train_norm,transformed_video_output_norm,self.batch_size)  # batch batch 2*conmmon_space_dim\n",
    "            sim_score_mat_train = vs_multilayer(cross_modal_vec_train, \"vs_multilayer_lt\",self.mpl_hidden)\n",
    "            sim_score_mat_train = tf.reshape(sim_score_mat_train, [self.batch_size,self.batch_size,3])#(30,30,3)\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            \n",
    "            \n",
    "            print (\"Building test network...............................\\n\")\n",
    "            \n",
    "            \n",
    "            # text_seq has shape [T, N] and embedded_seq has shape [self.test_batch_size, T, D].[1,10,300]\n",
    "            embedded_seq_test = tf.nn.embedding_lookup(embedding_mat, text_feature_test)#[1,10,300]\n",
    "            # 1. Encode the sentence into a vector representation, using the final\n",
    "            # hidden states in a one-layer bidirectional LSTM network\n",
    "            q_reshape_test = self.bilstm(embedded_seq_test)\n",
    "            \n",
    "            \n",
    "            # 1.1 noun_word(1,10)  embedding(1,10,300)\n",
    "            embedded_noun_word_test = tf.nn.embedding_lookup(embedding_mat,noun_word_feature_test)\n",
    "            #GRU encode \n",
    "            #embedded_noun_word_test = tf.convert_to_tensor(embedded_noun_word_test,dtype = tf.float32)\n",
    "            encoded_vector_test = CuDNNGRU(512,return_state=False)(embedded_noun_word_test)#1,512\n",
    "            embedded_noun_word_test = encoded_vector_test #(1,512)\n",
    "            # 1.2 fast_rcnn_vec(batch,64,2048)\n",
    "            #fast_rcnn_feature_train = tf.convert_to_tensor(fast_rcnn_feature_train,dtype = tf.float32)\n",
    "            #video_vector_test = CuDNNGRU(512,return_state=False,return_sequences=True)(fast_rcnn_feature_test)#(1, 64, 512)\n",
    "            video_vector_test = CuDNNGRU(512,return_state=False)(fast_rcnn_feature_test)\n",
    "            # 1.3 top_down_attention\n",
    "            #video_output_test = top_down_attention(video_vector_test,embedded_noun_word_test)#(1, 512)\n",
    "            video_output_test = video_vector_test\n",
    "            print (video_output_test.get_shape().as_list())\n",
    "                  \n",
    "            \n",
    "            # 2. three attention units over the words in each sentence\n",
    "            # 2. attention units over the words in each sentence fc(fc(Q+C+PRE+POST))\n",
    "            q_reshape_flat = tf.reshape(q_reshape_test, [self.n_step_text * self.test_batch_size, self.n_hidden_text * 2])#(10,2000)\n",
    "            visual_feature_test=tf.transpose(visual_feature_test, [0, 2, 1])  # batch ctx fea   (1,3,4096)\n",
    "            visual_test=tf.reshape(visual_feature_test,[self.test_batch_size*(2*self.context_num+1),self.visual_feature_dim]) #(3,4096)\n",
    "            query_term=fc('q2s_lt', q_reshape_flat, output_dim=self.semantic) #(10,2024)\n",
    "            moment_term=fc('c2s_lt', visual_test, output_dim=self.semantic)  #(3,1024)\n",
    "            \n",
    "            query_term=tf.reshape(query_term,[self.test_batch_size,self.n_step_text,self.semantic])  #(1,10,1024)\n",
    "            moment_term=tf.reshape( moment_term,[self.test_batch_size,2*self.context_num+1,self.semantic])#(1,3,1024)\n",
    "            term2=tf.reduce_sum(moment_term,1,keep_dims=True) #(1,1,1024)\n",
    "            term2=tf.reshape(term2,[self.test_batch_size,self.semantic]) #(1,1024)\n",
    "            term2=tf.reshape(tf.tile(term2,[1,self.n_step_text]),[self.test_batch_size,self.n_step_text,self.semantic]) #(1,10,1024)\n",
    "            term=tf.nn.relu(tf.reshape(tf.add(query_term,term2), [self.n_step_text * self.test_batch_size, self.semantic])) #(10,1024)\n",
    "            scores_obj1 = fc('fc_scores_query_lt',term, output_dim=1) #(10,1)\n",
    "            scores_obj1=tf.reshape(scores_obj1,[self.test_batch_size,self.n_step_text]) #(1,10)\n",
    "            is_not_pad=tf.cast(tf.not_equal(text_feature_test,0),tf.float32) \n",
    "            #probs_obj1=tf.nn.softmax(scores_obj1)\n",
    "            #probs_obj1=tf.multiply(probs_obj1,is_not_pad)\n",
    "            probs_obj1 = tf.multiply(scores_obj1,is_not_pad)\n",
    "            probs_obj1 = probs_obj1 / tf.reduce_sum(probs_obj1, 1, keep_dims=True)\n",
    "            temp1= tf.transpose(tf.reshape(tf.tile(probs_obj1, [1, self.n_input_text]),[self.test_batch_size, self.n_input_text, self.n_step_text]),[0,2,1])  # [N,T,embed_dim]\n",
    "            BoW_obj1 = tf.reduce_sum(tf.multiply(temp1,embedded_seq_test), reduction_indices=1)\n",
    "            print (BoW_obj1.get_shape().as_list())  #(1,300)\n",
    "            \n",
    "            input_vision_obj1=tf.reshape(visual_feature_test,[self.test_batch_size,-1]) #(1,12280)\n",
    "            # cross-modal part\n",
    "            transformed_clip_test_norm = tf.nn.l2_normalize(input_vision_obj1 , dim=1)\n",
    "            transformed_obj1_sent_test_norm = tf.nn.l2_normalize(BoW_obj1, dim=1)\n",
    "            transformed_video_output_test_norm = tf.nn.l2_normalize(video_output_test, dim=1)#(1,512)\n",
    "            \n",
    "            \n",
    "            \n",
    "            cross_modal_vec_test = self.cross_modal_comb(transformed_clip_test_norm,transformed_obj1_sent_test_norm,transformed_video_output_test_norm,self.test_batch_size)  # batch batch 2*conmmon_space_dim\n",
    "            sim_score_mat_test =vs_multilayer(cross_modal_vec_test, \"vs_multilayer_lt\",self.mpl_hidden,reuse=True)\n",
    "            sim_score_mat_test = tf.reshape(sim_score_mat_test, [3])\n",
    "            return sim_score_mat_train, sim_score_mat_test #(30,30,3)  (1,1,3)\n",
    "\n",
    "        \n",
    "    '''\n",
    "    compute alignment and regression loss\n",
    "    \n",
    "    '''\n",
    "    def compute_loss_reg(self, sim_reg_mat, offset_label):\n",
    "\n",
    "        sim_score_mat, p_reg_mat, l_reg_mat = tf.split(sim_reg_mat, 3, 2)\n",
    "        sim_score_mat = tf.reshape(sim_score_mat, [self.batch_size, self.batch_size])\n",
    "        l_reg_mat = tf.reshape(l_reg_mat, [self.batch_size, self.batch_size])\n",
    "        p_reg_mat = tf.reshape(p_reg_mat, [self.batch_size, self.batch_size])\n",
    "        # unit matrix with -2\n",
    "        I_2 = tf.diag(tf.constant(-2.0, shape=[self.batch_size]))\n",
    "        all1 = tf.constant(1.0, shape=[self.batch_size, self.batch_size])\n",
    "        #               | -1  1   1...   |\n",
    "\n",
    "        #   mask_mat =  | 1  -1  -1...   |\n",
    "\n",
    "        #               | 1   1  -1 ...  |\n",
    "        mask_mat = tf.add(I_2, all1)#shape=(30, 30)\n",
    "        # loss cls, not considering iou\n",
    "        I = tf.diag(tf.constant(1.0, shape=[self.batch_size]))\n",
    "        #I_half = tf.diag(tf.constant(0.5, shape=[self.batch_size]))\n",
    "        batch_para_mat = tf.constant(self.alpha, shape=[self.batch_size, self.batch_size])\n",
    "        para_mat = tf.add(I,batch_para_mat)\n",
    "        loss_mat = tf.log(tf.add(all1, tf.exp(tf.multiply(mask_mat, sim_score_mat))))\n",
    "        loss_mat = tf.multiply(loss_mat, para_mat)\n",
    "        loss_align = tf.reduce_mean(loss_mat)\n",
    "        # regression loss\n",
    "        l_reg_diag = tf.matmul(tf.multiply(l_reg_mat, I), tf.constant(1.0, shape=[self.batch_size, 1]))\n",
    "        p_reg_diag = tf.matmul(tf.multiply(p_reg_mat, I), tf.constant(1.0, shape=[self.batch_size, 1]))\n",
    "        offset_pred = tf.concat((p_reg_diag, l_reg_diag),1)\n",
    "        loss_reg = tf.reduce_mean(tf.abs(tf.subtract(offset_pred, offset_label)))\n",
    "\n",
    "        loss=tf.add(tf.multiply(self.lambda_regression, loss_reg), loss_align)#L=L(align)+lambda*L(location)\n",
    "        return loss, offset_pred, loss_reg\n",
    "\n",
    "\n",
    "    def init_placeholder(self):\n",
    "        visual_featmap_ph_train = tf.placeholder(tf.float32, shape=(self.batch_size, self.visual_feature_dim,2 * self.context_num + 1))  # (50,4096,3)\n",
    "        sentence_ph_train = tf.placeholder(tf.int32, shape=(self.batch_size,self.n_step_text ))#(30,10)\n",
    "        offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))#30,2\n",
    "        \n",
    "        fast_rcnn_vec_train = tf.placeholder(tf.float32, shape=(self.batch_size, 64,2048))#(30,64,2048)\n",
    "        noun_word_ph_train = tf.placeholder(tf.int32, shape=(self.batch_size,self.n_step_text ))#(30,10)\n",
    "        \n",
    "        visual_featmap_ph_test = tf.placeholder(tf.float32, shape=(self.test_batch_size, self.visual_feature_dim,2 * self.context_num + 1))  # input feature: current clip, pre-contex, and post contex （1,4096,3）\n",
    "        sentence_ph_test = tf.placeholder(tf.int32, shape=(self.test_batch_size,self.n_step_text ))#batch_size=1,T=10 （1,10）\n",
    "        \n",
    "        fast_rcnn_vec_test = tf.placeholder(tf.float32, shape=(self.test_batch_size, 64,2048))\n",
    "        noun_word_ph_test = tf.placeholder(tf.int32, shape=(self.test_batch_size,self.n_step_text ))#(1,10)\n",
    "  \n",
    "        \n",
    "        return visual_featmap_ph_train, sentence_ph_train, offset_ph,fast_rcnn_vec_train, noun_word_ph_train,visual_featmap_ph_test, sentence_ph_test,fast_rcnn_vec_test,noun_word_ph_test\n",
    "\n",
    "    \n",
    "    def get_variables_by_name(self,name_list):\n",
    "        v_list = tf.trainable_variables()\n",
    "        v_dict = {}\n",
    "        for name in name_list:\n",
    "            v_dict[name] = []\n",
    "        for v in v_list:\n",
    "            for name in name_list:\n",
    "                if name in v.name: v_dict[name].append(v)\n",
    "\n",
    "        for name in name_list:\n",
    "            print (\"Variables of <\"+name+\">\")\n",
    "            for v in v_dict[name]:\n",
    "                print (\"    \"+v.name)\n",
    "        return v_dict\n",
    "\n",
    "    def training(self, loss):\n",
    "\n",
    "        v_dict = self.get_variables_by_name([\"lt\"])\n",
    "        vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')\n",
    "        vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict[\"lt\"])\n",
    "        return vs_train_op\n",
    "\n",
    "\n",
    "    def construct_model(self):\n",
    "        # initialize the placeholder\n",
    "        self.visual_featmap_ph_train, self.sentence_ph_train,self.offset_ph,self.fast_rcnn_vec_train,self.noun_word_ph_train,self.visual_featmap_ph_test, self.sentence_ph_test,self.fast_rcnn_vec_test,self.noun_word_ph_test=self.init_placeholder()#train:(50,4096,3)  (50,10) (50,2)  test:(1,4096,3) (1,10)\n",
    "        # build inference network\n",
    "        sim_reg_mat, sim_reg_mat_test= self.visual_semantic_infer(self.visual_featmap_ph_train, self.sentence_ph_train,self.fast_rcnn_vec_train,self.noun_word_ph_train, self.visual_featmap_ph_test, self.sentence_ph_test,self.fast_rcnn_vec_test,self.noun_word_ph_test)\n",
    "        # compute loss\n",
    "        self.loss_align_reg, offset_pred, loss_reg = self.compute_loss_reg(sim_reg_mat, self.offset_ph)\n",
    "        # optimize\n",
    "        self.vs_train_op = self.training(self.loss_align_reg)\n",
    "        return self.loss_align_reg, self.vs_train_op, sim_reg_mat_test, offset_pred, loss_reg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense_to_one_hot(labels_dense, num_classes):\n",
    "    \"\"\"Convert class labels from scalars to one-hot vectors.\"\"\"\n",
    "    num_labels = labels_dense.shape[0]\n",
    "    index_offset = np.arange(num_labels) * num_classes\n",
    "    labels_one_hot = np.zeros((num_labels, num_classes))\n",
    "    labels_one_hot.flat[index_offset+labels_dense.ravel()] = 1\n",
    "    return labels_one_hot\n",
    "\n",
    "\n",
    "def calculate_IoU(i0,i1):\n",
    "    union = (min(i0[0], i1[0]), max(i0[1], i1[1]))\n",
    "    inter = (max(i0[0], i1[0]), min(i0[1], i1[1]))\n",
    "    iou = 1.0*(inter[1]-inter[0])/(union[1]-union[0])\n",
    "    return iou\n",
    "\n",
    "def nms_temporal(x1,x2,s, overlap):\n",
    "    pick = []\n",
    "    assert len(x1)==len(s)\n",
    "    assert len(x2)==len(s)\n",
    "    if len(x1)==0:\n",
    "        return pick\n",
    "\n",
    "    #x1 = [b[0] for b in boxes]\n",
    "    #x2 = [b[1] for b in boxes]\n",
    "    #s = [b[-1] for b in boxes]\n",
    "    union = list(map(operator.sub, x2, x1)) # union = x2-x1\n",
    "    I = [i[0] for i in sorted(enumerate(s), key=lambda x:x[1])] # sort and get index\n",
    "\n",
    "    while len(I)>0:\n",
    "        i = I[-1]\n",
    "        pick.append(i)\n",
    "        xx1 = [max(x1[i],x1[j]) for j in I[:-1]]\n",
    "        xx2 = [min(x2[i],x2[j]) for j in I[:-1]]\n",
    "        inter = [max(0.0, k2-k1) for k1, k2 in zip(xx1, xx2)]\n",
    "        o = [inter[u]/(union[i] + union[I[u]] - inter[u]) for u in range(len(I)-1)]\n",
    "        I_new = []\n",
    "        for j in range(len(o)):\n",
    "            if o[j] <=overlap:\n",
    "                I_new.append(I[j])\n",
    "        I = I_new\n",
    "    return pick\n",
    "\n",
    "'''\n",
    "compute recall at certain IoU\n",
    "'''\n",
    "def compute_IoU_recall_top_n_forreg(top_n, iou_thresh, sentence_image_mat, sentence_image_reg_mat, sclips, iclips,movie_clip_sentences):\n",
    "    correct_num = 0.0\n",
    "    for k in range(sentence_image_mat.shape[0]):\n",
    "        gt = sclips[k]\n",
    "        gt_start = float(gt.split(\"_\")[1])\n",
    "        gt_end = float(gt.split(\"_\")[2].split('.')[0])\n",
    "        #print gt +\" \"+str(gt_start)+\" \"+str(gt_end)\n",
    "        sim_v = [v for v in sentence_image_mat[k]]\n",
    "        starts = [s for s in sentence_image_reg_mat[k,:,0]]\n",
    "        ends = [e for e in sentence_image_reg_mat[k,:,1]]\n",
    "        picks = nms_temporal(starts,ends, sim_v, iou_thresh-0.05)\n",
    "        if top_n<len(picks): picks=picks[0:top_n]\n",
    "        for index in picks:\n",
    "            pred_start = sentence_image_reg_mat[k, index, 0]\n",
    "            pred_end = sentence_image_reg_mat[k, index, 1]\n",
    "            iou = calculate_IoU((gt_start, gt_end),(pred_start, pred_end))\n",
    "            if iou>=iou_thresh:\n",
    "                correct_num+=1\n",
    "                break\n",
    "    return correct_num\n",
    "\n",
    "'''\n",
    "evaluate the model\n",
    "'''\n",
    "def do_eval_slidingclips(sess, vs_eval_op, model, movie_length_info, iter_step,context_num):\n",
    "    IoU_thresh = [0.1,0.3, 0.5,0.7,0.9]\n",
    "    all_correct_num_10 = [0.0]*5\n",
    "    all_correct_num_5 = [0.0]*5\n",
    "    all_correct_num_1 = [0.0]*5\n",
    "    all_retrievd = 0.0\n",
    "    for movie_name in model.test_set.movie_names:\n",
    "        print (\"Test movie: \"+movie_name+\"....loading movie data\")\n",
    "        movie_clip_featmaps, movie_clip_sentences=model.test_set.load_movie_slidingclip(movie_name, 16)#sample = 16\n",
    "        print (\"sentences: \"+ str(len(movie_clip_sentences)))\n",
    "        print (\"clips: \"+ str(len(movie_clip_featmaps)))\n",
    "        sentence_image_mat=np.zeros([len(movie_clip_sentences), len(movie_clip_featmaps)])\n",
    "        sentence_image_reg_mat=np.zeros([len(movie_clip_sentences), len(movie_clip_featmaps), 2])\n",
    "        for k in range(len(movie_clip_sentences)):\n",
    "           \n",
    "            sent_vec=movie_clip_sentences[k][1]\n",
    "            sent_vec = preprocess_sentence(sent_vec, vocab_dict, T)\n",
    "            sent_vec=np.reshape(sent_vec,[1,T])#(1,T)\n",
    "            \n",
    "            noun_word_vec = movie_clip_sentences[k][2]\n",
    "            noun_word_vec = preprocess_sentence(noun_word_vec, vocab_dict, T)\n",
    "            noun_word_vec = np.reshape(noun_word_vec,[1,T])#(1,T)\n",
    "            \n",
    "            \n",
    "            for t in range(len(movie_clip_featmaps)):\n",
    "                featmap = movie_clip_featmaps[t][1] #(4096,3)\n",
    "                visual_clip_name = movie_clip_featmaps[t][0]\n",
    "                start = float(visual_clip_name.split(\"_\")[1])\n",
    "                end = float(visual_clip_name.split(\"_\")[2].split(\".\")[0])\n",
    "                featmap = np.reshape(featmap, [ 1, featmap.shape[0],2*context_num+1]) #1, 4096, 3\n",
    "                \n",
    "                fast_rcnn_fea = movie_clip_featmaps[t][2] #(64,2048)\n",
    "                fast_rcnn_fea =  np.reshape(fast_rcnn_fea, [1,64,2048]) \n",
    "                \n",
    "                \n",
    "                feed_dict = {\n",
    "                model.visual_featmap_ph_test: featmap,\n",
    "                model.sentence_ph_test:sent_vec,\n",
    "                    \n",
    "                model.fast_rcnn_vec_test:fast_rcnn_fea,\n",
    "                model.noun_word_ph_test:noun_word_vec\n",
    "                    \n",
    "                }\n",
    "                outputs = sess.run(vs_eval_op,feed_dict=feed_dict)\n",
    "                sentence_image_mat[k,t] = outputs[0]\n",
    "                reg_end = end+outputs[2]\n",
    "                reg_start = start+outputs[1]\n",
    "\n",
    "                sentence_image_reg_mat[k,t,0] = reg_start\n",
    "                sentence_image_reg_mat[k,t,1] = reg_end\n",
    "        iclips = [b[0] for b in movie_clip_featmaps]\n",
    "        sclips = [b[0] for b in movie_clip_sentences]\n",
    "\n",
    "        # calculate Recall@m, IoU=n\n",
    "        for k in range(len(IoU_thresh)):\n",
    "            IoU=IoU_thresh[k]\n",
    "            correct_num_10 = compute_IoU_recall_top_n_forreg(10, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips,movie_clip_sentences)\n",
    "            correct_num_5 = compute_IoU_recall_top_n_forreg(5, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips,movie_clip_sentences)\n",
    "            correct_num_1 = compute_IoU_recall_top_n_forreg(1, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips,movie_clip_sentences)\n",
    "            #print (movie_name+\" IoU=\"+str(IoU)+\", R@10: \"+str(correct_num_10/len(sclips))+\"; IoU=\"+str(IoU)+\", R@5: \"+str(correct_num_5/len(sclips))+\"; IoU=\"+str(IoU)+\", R@1: \"+str(correct_num_1/len(sclips)))\n",
    "            all_correct_num_10[k]+=correct_num_10\n",
    "            all_correct_num_5[k]+=correct_num_5\n",
    "            all_correct_num_1[k]+=correct_num_1\n",
    "        all_retrievd+=len(sclips)\n",
    "    for k in range(len(IoU_thresh)):\n",
    "        print (\" IoU=\"+str(IoU_thresh[k])+\", R@10: \"+str(all_correct_num_10[k]/all_retrievd)+\"; IoU=\"+str(IoU_thresh[k])+\", R@5: \"+str(all_correct_num_5[k]/all_retrievd)+\"; IoU=\"+str(IoU_thresh[k])+\", R@1: \"+str(all_correct_num_1[k]/all_retrievd))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run_training():\n",
    "    max_steps=50000\n",
    "    lr=0.01\n",
    "    current_lr=lr\n",
    "    train_batch_size=30\n",
    "    display_step=5\n",
    "    test_iter=5000\n",
    "    semantic_size=1024\n",
    "    mpl_hidden=1000\n",
    "    context_num=1\n",
    "    visual_feature_dim=4096\n",
    "    embed_dim = 300\n",
    "    lstm_dim = 1000\n",
    "    train_csv_path = \"./charades_300/clip_sentence_pairs_iou_charades_noun_word.pkl\"\n",
    "    test_csv_path = \"./charades_300/charades_clip_sentence_test_new_noun_word.pkl\"\n",
    "    train_feature_dir = \"./charades_C3D/charades_16_32_48_64_overlap0.8_c3d_fc6_train_pool/\"\n",
    "    test_feature_dir=\"./charades_C3D/charades_16_32_overlap0.8_c3d_fc6_test_pool/\"\n",
    "    train_fast_rcnn_path='./charades_fastrcnn/Charades_train_video_pics_vec_pool/'\n",
    "    test_fast_rcnn_path='./charades_fastrcnn/Charades_test_video_pics_vec_pool/'\n",
    "    \n",
    "    \n",
    "    \n",
    "    model = SLAT_Model(train_batch_size, train_csv_path, test_csv_path , lr, visual_feature_dim, embed_dim,lstm_dim, T,train_feature_dir,test_feature_dir,semantic_size,mpl_hidden,train_fast_rcnn_path,test_fast_rcnn_path)\n",
    "    \n",
    "    with tf.Graph().as_default():\n",
    "\n",
    "        loss_align_reg, vs_train_op, vs_eval_op, offset_pred, loss_reg = model.construct_model()\n",
    "        # Create a session for running Ops on the Graph.\n",
    "        sess = tf.Session()\n",
    "        # Run the Op to initialize the variables.\n",
    "        init = tf.global_variables_initializer()\n",
    "        sess.run(init)\n",
    "        #max_steps=100000\n",
    "        for step in xrange(max_steps):\n",
    "            start_time = time.time()\n",
    "            feed_dict = model.fill_feed_dict_train_reg()\n",
    "            _, loss_value, offset_pred_v, loss_reg_v = sess.run([vs_train_op, loss_align_reg, offset_pred, loss_reg], feed_dict=feed_dict)\n",
    "            duration = time.time() - start_time\n",
    "\n",
    "            if step % display_step == 0:\n",
    "                # Print status to stdout.\n",
    "                print('Step %d: loss = %.3f (%.3f sec)' % (step, loss_value, duration))\n",
    "            if (step+1)%50000==0:\n",
    "                current_lr=current_lr*0.1\n",
    "                model.vs_lr=current_lr\n",
    "                print (model.vs_lr)\n",
    "            if (step+1) % test_iter == 0:\n",
    "                print (\"Start to test:-----------------\\n\")\n",
    "                movie_length_info=pickle.load(open(\"./video_allframes_info_charades_new.pkl\",'rb'))\n",
    "                do_eval_slidingclips(sess, vs_eval_op, model, movie_length_info, step+1,context_num)\n",
    "\n",
    "def main(_):\n",
    "    run_training()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    tf.app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}