conv_layer = tf.keras.layers.Conv1D(
    filters=64, 
    kernel_size=2,
    dilation_rate=4, 
    padding='causal')

Calling this layer will preserve the temporal dimension of the input by adding left padding which is not always desirable. As we stack more and more layers with larger dilation rate padding will become a large portion of the input data. Also the default implementation of dilated convolution layer implicitly define some padding based on your first input restricting your choice of the input shape later on.

x = np.random.randn(1, 150, 96).astype(np.float32)
conv_layer(x).shape
--> TensorShape([1, 150, 64])

Now if you try to feed some other data with different shape you’ll get an error:

y = np.random.randn(1, 140, 96).astype(np.float32)
conv_layer(y)
--> InvalidArgumentError: padded_shape[0]=146 is not divisible by block_shape[0]=4 [Op:SpaceToBatchND]

We can handle dilation ourselves as we want in DilatedConv1D class that inherits from the built-in Conv1D layer.

import tensorflow as tf
import numpy as np
from tensorflow.keras import layers

def time_to_batch(value, dilation, name=None):
    shape = tf.shape(value)
    pad_elements = dilation - 1 - (shape[1] + dilation - 1) % dilation
    padded = tf.pad(value, [[0, 0], [0, pad_elements], [0, 0]])
    reshaped = tf.reshape(padded, [-1, dilation, shape[2]])
    transposed = tf.transpose(reshaped, perm=[1, 0, 2])
    return tf.reshape(transposed, [shape[0] * dilation, -1, shape[2]]), pad_elements


def batch_to_time(value, dilation, name=None):
    shape = tf.shape(value)
    prepared = tf.reshape(value, [dilation, -1, shape[2]])
    transposed = tf.transpose(prepared, perm=[1, 0, 2])
    return tf.reshape(transposed,
                      [tf.divide(shape[0], dilation), -1, shape[2]])


class DilatedConv1D(tf.keras.layers.Conv1D):
    
    def __init__(self, nb_filters, kernel_size, dilation_rate=1, **kwargs):
        super().__init__(nb_filters, kernel_size, dilation_rate=1, **kwargs)
        self.custom_dilation = dilation_rate if isinstance(dilation_rate, tuple) else (dilation_rate, )
    
    def call(self, input_tensor, **kwargs):
        if self.custom_dilation [0] > 1:
            x, pad = time_to_batch(input_tensor, self.custom_dilation[0])
            x = super().call(x)
            output = batch_to_time(x, self.custom_dilation[0])
            width = tf.shape(output)[1] - pad
            output = output[:,:width]
        else:
            output = super().call(input_tensor)
        return output

Now we can implement CausalConv1D on top of DilatedConv1D so that we will be able to handle varying input shapes and control whether we want causal left padding or not with TF-like padding keywords: “valid” and “same”.

class CausalConv1D(DilatedConv1D):
    
    def __init__(self, nb_filters, kernel_size, padding='valid', **kwargs):
        allowed_paddings = ['same', 'valid']
        if padding not in allowed_paddings:
            raise ValueError('Unknown padding, allowed: %s' % str(allowed_paddings))
        super().__init__(nb_filters, kernel_size, **kwargs)
        self.causal_padding = padding

    def call(self, input_tensor,  **kwargs):
        pad = self.custom_dilation[0] * (self.kernel_size[0] - 1)
        if self.causal_padding == 'same':
            causal_padding = [[0, 0], [pad, 0], [0, 0]]
            padded_tensor = tf.pad(input_tensor, paddings=causal_padding)
            output = super().call(padded_tensor)
        else: 
            out_width = tf.shape(input_tensor)[1] - pad
            output = super().call(input_tensor)[:, :out_width]
            
        return output
    

Let’s try our new causal layer in action. We’ll create two identical architectures but with different causal layer implementations and since we set the same weights for both networks they should return the same result.

class OriginalNetwork(tf.keras.Model):
    def __init__(self):
        super(OriginalNetwork, self).__init__()

        self.layer_stack = [
            tf.keras.layers.Conv1D(32, kernel_size=3, padding='causal', activation='relu', dilation_rate=1),
            tf.keras.layers.Conv1D(32, kernel_size=3, padding='causal', activation='relu', dilation_rate=2),
            tf.keras.layers.Conv1D(32, kernel_size=3, padding='causal', activation='relu', dilation_rate=4),
            tf.keras.layers.Conv1D(1, kernel_size=3, padding='causal', dilation_rate=8)
        ]

    def call(self, input_tensor, **kwargs):
        x = input_tensor
        for layer in self.layer_stack:
            x = layer(x, **kwargs)
        return x

    
class ModifiedNetwork(tf.keras.Model):
    def __init__(self, padding='valid'):
        super(ModifiedNetwork, self).__init__()

        self.layer_stack = [
            CausalConv1D(32, kernel_size=3, activation='relu', padding=padding, dilation_rate=1),
            CausalConv1D(32, kernel_size=3, activation='relu', padding=padding, dilation_rate=2),
            CausalConv1D(32, kernel_size=3, activation='relu', padding=padding, dilation_rate=4),
            CausalConv1D(1, kernel_size=3, padding=padding, dilation_rate=8)
        ]
            

    def call(self, input_tensor, training=True, **kwargs):
        x = input_tensor
        for layer in self.layer_stack:
            x = layer(x, **kwargs)
        return x
    

x = np.random.randn(1, 150, 96).astype(np.float32)
net1 = OriginalNetwork()
net2 = ModifiedNetwork(padding='same')
_, _ = net1(x), net2(x)
net1.set_weights(net2.get_weights())
output1, output2 = net1(x), net2(x)
print('Shapes are equal: ' + str((output1.shape == output2.shape)))
print('Values are equal: ' + str((output1 == output2).numpy().all()))

Outputs:

Shapes are equal: True
Values are equal: True

With valid padding the modified network should output reduced temporal dimension.

x = np.random.randn(1, 150, 96).astype(np.float32)
net1 = OriginalNetwork()
net2 = ModifiedNetwork(padding='valid')
_, _ = net1(x), net2(x)
net1.set_weights(net2.get_weights())
output1, output2 = net1(x), net2(x)
print('Shapes are: ' + str((output1.shape, output2.shape)))
print('Values are equal: ' + str((output1[0,30:] == output2[0]).numpy().all()))

Outputs:

Shapes are: (TensorShape([1, 150, 1]), TensorShape([1, 120, 1]))
Values are equal: True

FB algorithm heavily relies on Markov property and output independence assumptions from HMM. Actually a lot of terms here are inherited from HMM, so if you’re not familliar then this HMM tutorial might come in handy.

Suppose that \(z=\{z_1,z_2,..,z_n\}\) is hidden variable, $x={z_1,z_2,..,z_n}$ is observable variable and $k=1..n$ denotes timestep, then

• \(p(x_k\vert z_k)\) is an Emission probability;
• $p(z_k\vert z_{k-1})$ is a Transition probability;
• $p(z_1)$ is an Initial probability.

To simplify the notation let’s further assume that hidden state variable $z_i$ defined on an interger set ${1,..,m}$

The goal of FB algorithm is to compute $p(z_k\vert x)$.

The forward pass computes: $p(z_k,x_{1:k}) \forall k=1,..,n$

The backward pass computes: $p(x_{k+1:n} \vert z_k) \forall k=1,..,n$

For now let’s assume that we’ve already computed forward and backward parts, in other words we know $p(z_k,x_{1:k})$ and $p(x_{k+1:n} \vert z_k)$. How does it help us to get desired $p(z_k\vert x)$?

The last step applies D-separation rule ($x_{k+1:n}$ is conditionally independent of $x_{1:k}$ given $z_k$). After simplification we have:

And we already know that $p(z_k, x_{1:k}), p(x_{k+1:n}\vert z_k)$ can be found with forward and backward algorithms. The proportionality is instead of equality is due to omitting normalizing constant $p(x)$ which we can compute by marginalizing $p(x,z_k)$ over all finite set of $z_k$: \(p(x) = \sum_{z_{k}=1}^{m} p(x,z_k)\)

Forward algorithm

• $p(x_k\vert z_k)$ is given as an emission probability
• $p(z_k\vert z_{k-1})$ is given as a transition probability
• $p(z_{k-1},x_{1:k-1})$ is unknown, but you can notice the recurrence here if you’ll look at what we are trying to compute: $p(z_k, x_{1:k})$

To find $p(z_k, x_{1:k})$ we need to compute $p(z_{k-1},x_{1:k-1})$, for $p(z_{k-1},x_{1:k-1})$ you have to compute $p(z_{k-2},x_{1:k-2})$ and so on until $p(z_{1},x_{1})$. And $p(z_{1},x_{1})$ is just an emission probability that is known. Now we know all the parts to achieve our goal finding $p(z_k, x_{1:k})$.

What about time complexity? We have $n$ timesteps ($k=1..n$) and inside each timestep we iterate over $m$ values of $z_k$ and $m$ values of $z_{k-1}$, this gives us $O(nm^2)$ time complexity.

Backward algorithm

• $p(x_{k+2:n} \vert z_{k+1})$ can be computed recurrently
• $p(x_{k+1}\vert z_{k+1})$ is given as an emission probability
• $p(z_{k+1}\vert z_{k})$ is given as a transition probability

We compute $p(x_{k+2:n} \vert z_{k+1})$ recurrently until we reach $p(x_n\vert z_{n-1})$. We can do the same routine as previously:

This formula is a special case of previous more general formula where the term $p(x_{k+2:n} \vert z_{k+1})$ has become equal to 1, and if $k=n-1$ theh we’ll get $p(x_{n+1} \vert z_n) = 1$.

Summarizing all that in the final formula:

This is quite similar to forward part and time complexity is also $O(nm^2)$ because we have $k=1..n$ timesteps, $m$ values of $z_k$ and $z_{k+1}$ that we iterate through.

Naive approach

To better understand the value of FB algorithm let’s just compare it with the naive approach. With conditional probability formula we know that:

We can try directly compute $p(x,z_k)$ and for that we have to take into account all possible sequences of $z$ where $z_k$ might have been occured or in other words we should marginalize out all except $z_k$: $\hat{z} = z_1..z_{k-1},z_{k+1}..z_n$ variables:

The number of sums is equal to number of possible permutations with repetitions of hidden state sequence $\hat{z}$ which is equal to $m^{n-1}$. This is an exponential time complexity, for $T = 101$ steps and $m = 10$ hidden states it will have $10^{100}$ terms to compute. While Forward-Backward algorithm will have $m^2 \times n = 100\times101 \approx 10^4$, that is $10^{96}$ times faster!

def encode(url):
    img = open_image(url)
    img = preprocess(img)
    data = np.asarray([img])
    if net.blobs['data'].data.shape[0] != 1:
        net.blobs['data'].reshape(1,3,224,224)
    result = net.forward(data=data)
    return net.blobs['pool5'].data[0].flatten().copy()

Encode your documents and queries and perform nearest neighbor (NN) search. For fast NN search I put vectors into Annoy index.

    
def search(query_vector):
    # get 9 ids of nearest vectors from index
    ids = index.get_nns_by_vector(query_vector, 9)
    # load images by ids and show them
    images=[]
    for i in ids:
        img = Image.open(id2file[i])
        if img is None:
            continue
        img = centeredCrop(np.array(resize(img, 128, 128)), 128, 128)
        images.append(img)
    plt.figure(2, figsize=(10,10))
    show(np.array(images))

search(sea)

search(sea+woman)

search(building + crowd*0.5 + sea*1.5)

search(woman + car*3 + dress)

search(coffee*1.3 + burger)

search(man + dress)

search(woman_in_dress)

search(woman_in_dress - dress)

Query “extreme”:

On the left is the query result and on the right - some groups of images. It’s clear that pictures in the same group are closer in semantic than images across. Here we got two clusters, first for the American rock band and the second for some extreme activity.

Query “destruction”:

Now we have three clusters and arguably three slightly different meanings: an explosion, destroyed buildings and a forest fire. Let’s try a query that for sure should show images with different semantic.

Query “apple”:

As you might guess Google will return results containing some Apple products and we can automatically move images with different meaning to their semantic group by applying clustering algorithm on CNN codes. Having multiple images in top-k biggest clusters we can pull out centroids and use them as different representation of the same concept. Let’s do this for our examples:

Here centroids of the “distruction” clusters can be thought as different representations of a destruction. There are many other applications, for example, semantic clusters can help to find the most popular meaning for some text term. We can also apply clustering to filter duplicates as they tend to fall into the same group, in this case we need to select images from different groups.

So coming to conclusion, deep convolutional neural networks can produce very good abstractions that can be further used in many visual recognition tasks that hardly be possible without them.

Sport cars

Race cars

Car wheels

Now if you look at the next examples, you will find that they are pretty ambigous if you are asked to assign only one label, let’s say, out of five. And for a number of categories there a lot of images with such ambiguity.

The major problem with the dataset is that for many of the images we can’t say precisely if a category A or a category B describes it the best and the model in such situation is penalized during the training for the the mismatch between the predicted A category and B category given as a ground truth. So even if the image from the evaluation set is better in the sense of an ambiguity but still can be described with multiple labels, it will be harder for the network to determine a pravailing label because the training dataset doesn’t train it to do so very well. And this explain such a big margin between top-1 and top-5 errors.

To share it I’ve created a small script available on github. You can run it as in this example:

python create_multilabel_lmdb.py 
	--images /path/to/image_file/images.txt 
	--labels /path/to/labels_file/labels.npy 
	--imagesOut /path/to/image-lmdb 
	--labelsOut /path/to/label-lmdb 
	-n size_of_test_set 
	--maxPx 256 
	--minPx 227 
	--shuffle=true
	

What this script do is reads the images text file having the format like this:

 /path/to/dir/img1.jpg
 /path/to/dir/img2.jpg
 ...

Reads the labels file, which is just a 2d numpy array serialized using numpy, here is the example of an array:

[
	[1,0,1,0,1], 
	[0,1,0,0,1],
	[1,1,0,0,0]
]

The first row indicates that the first image (in the images text file) has labels 1,3 and 5. And the second row says that the second image has labels 2 and 5, the third row - labels 1 and 2. This script also shuffles the data, resizes images preserving an aspect ratio and prints mean image values at the end of the work. If error occures (e.g if some image file is corrupted or missing), the procedure skips the corrupted image and its label and continues the progress.

Sentiment analysis and opinion mining is the field of study that analyzes people’s opinions, sentiments, evaluations, attitudes, and emotions from written language.¹

So the the basic task of sentiment analysis is classifying text into some emotive categories. The most common set of categories are: positive, neutral and negative.

Methods for sentiment analysis

There is a number of methods for sentiment analysis that can be divided in two groups:

• Lexicon-based methods
• Machine learning methods
  • NB (Naive Bayes classifier)
  • biNB (Naive Bayes with bag of bigram features)
  • SVM (Support Vector Machine)
  • RNTN (Recursive Neural Tensor Network)
  • RNN (Recursive Neural Network)
  • MV-RNN (Matrix-Vector RNN)

The last method in the list is claimed to have the better performance than the others. ²

CoreNLP library

As you may already know there is great open source library for natural language processing named CoreNLP which includes RNTN for sentiment analysis. The library is written in Java, but there are also some wrappers for other languages.

How to use it

Let’s dive into details on how to use CoreNLP in Scala. Since Scala is compatible with Java, you can simply import CoreNLP library in your Scala project. To include library and models for sentiment component in your project add the following dependency to the sbt build file:

libraryDependencies += "edu.stanford.nlp" % "stanford-corenlp" % "3.3.1" artifacts (Artifact("stanford-corenlp", "models"), Artifact("stanford-corenlp"))

When doing analysis with CoreNLP you get fine grained predictions as a result (i.e. 5 classes: very positive, positive, neutral, negative, very negative) which are less accurate than Positive/Neutral/Negative ones. In the code below, we create three categories and map result into them in the end of getSentiment method.

import java.util.Properties
import edu.stanford.nlp.pipeline.StanfordCoreNLP
import edu.stanford.nlp.ling.CoreAnnotations
import edu.stanford.nlp.sentiment.SentimentCoreAnnotations
import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations

object SentimentCategory extends Enumeration {
  type SentimentCategory = Int
  val Negative = 0
  val Neutral = 1
  val Positive = 2
}

trait SentimentTools {
    val props = new Properties()
    props.setProperty("annotators", "tokenize, ssplit, parse, sentiment");
    val pipeline = new StanfordCoreNLP(props)

    def getSentiment(text : String): SentimentCategory = {
      var mainSentiment = 0
      if (text != null && text.length() > 0) {
        var longest = 0
        val annotation = pipeline.process(text)
        val list = annotation.get(classOf[CoreAnnotations.SentencesAnnotation])
        val it = list.iterator()
        while (it.hasNext)
        {
          val sentence = it.next()
          val tree = sentence.get(classOf[SentimentCoreAnnotations.AnnotatedTree])
          val sentiment = RNNCoreAnnotations.getPredictedClass(tree)
          val partText = sentence.toString()
          if (partText.length() > longest) {
            mainSentiment = sentiment
            longest = partText.length()
          }
        }
      }
      import SentimentCategory._
      if (mainSentiment < 2)
        Negative
      else if (mainSentiment == 2)
        Neutral
      else
        Positive
    }
}