Introduction

Visualize the boundary value after learning. It is as follows.

If the input dimension is two-dimensional, you can visualize the boundaries in this way. This time, I would like to visualize TensorFlow's multi-layer perceptron as an example.

Learning data
Model (multilayer perceptron)
Visualization policy 4.meshgrid
Contour plot
All chords History

1. Learning data

The data used this time uses the crescent-shaped data set in the scikit-learn dataset. Since it is two-dimensional data, it can be illustrated on a plane.

from sklearn import datasets
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
'''
Generate crescent data
'''
N = 300  #Specify the number of data
X, y = datasets.make_moons(N, noise=0.2)
Y = y.reshape(N, 1)

#Divide into training data and test data
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8)

'''
Data plot
'''
plt.figure(figsize=(10, 6))
plt.ylim([-1,1.5])
plt.xlim([-1.5,2.5])
plt.scatter(X[y==1][:,0], X[y==1][:,1], s=10, marker='.')
plt.scatter(X[y==0][:,0], X[y==0][:,1], s=10, marker='^')
plt.show()

Let's learn with this data and draw a border.

2. Model (multilayer perceptron)

Learning is done with a TensorFlow (1.x) multi-layer perceptron. The model is just an example, so it doesn't matter what model you use to draw the boundary values. Combined hidden layers. The code for the 3-layer perceptron is below.

class ThreelayerPerceptron(object):
    '''
Initialization
    '''
    def __init__(self, n_in, n_hidden, n_out):
        self.n_in = n_in
        self.n_hidden = n_hidden
        self.n_out = n_out
        self.weights = []
        self.biases = []

        self._x = None
        self._y = None
        self._t = None,
        self._sess = None
        self._history = {
            'accuracy': [],
            'loss': []
        }

    '''
weight
    '''
    def weight_variable(self, shape):
        initial = tf.truncated_normal(shape, stddev=0.01)
        return tf.Variable(initial)

    '''
bias
    '''
    def bias_variable(self, shape):
        initial = tf.zeros(shape)
        return tf.Variable(initial)

    '''
Model definition (3-layer perceptron)
    '''
    def inference(self, x):
        
        #Input layer-Hidden layer
        self.weights.append(self.weight_variable([self.n_in, self.n_hidden]))
        self.biases.append(self.bias_variable([self.n_hidden]))
        h = tf.nn.sigmoid(tf.matmul(x, self.weights[-1]) + self.biases[-1])

        #Hidden layer-Output layer
        self.weights.append(self.weight_variable([self.n_hidden, self.n_out]))
        self.biases.append(self.bias_variable([self.n_out]))
        y = tf.nn.sigmoid(tf.matmul(h, self.weights[-1]) + self.biases[-1])

        return y

    '''
Loss function
    '''
    def loss(self, y, t):
        cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)))
        return cross_entropy

    '''
Optimization algorithm
    '''
    def training(self, loss):
        optimizer = tf.train.GradientDescentOptimizer(0.05)
        train_step = optimizer.minimize(loss)
        return train_step

    '''
Correct answer rate
    '''
    def accuracy(self, y, t):
        correct_prediction = tf.equal(tf.cast(tf.greater(y, 0.5),tf.float32), t)
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        return accuracy
    
    '''
Forecast
    '''
    def prediction(self, y):
        return tf.cast(tf.greater(y, 0.5),tf.float32)

    '''
Learning
    '''
    def fit(self, X_train, Y_train,
            nb_epoch=100, batch_size=100, p_keep=0.5,
            verbose=1):
        x = tf.placeholder(tf.float32, shape=[None, self.n_in])
        t = tf.placeholder(tf.float32, shape=[None, self.n_out])

        self._x = x
        self._t = t

        y = self.inference(x)
        loss = self.loss(y, t)
        train_step = self.training(loss)
        accuracy = self.accuracy(y, t)

        init = tf.global_variables_initializer()
        sess = tf.Session()
        sess.run(init)

        self._y = y
        self._sess = sess

        N_train = len(X_train)
        n_batches = N_train // batch_size

        for epoch in range(nb_epoch):
            X_, Y_ = shuffle(X_train, Y_train)

            for i in range(n_batches):
                start = i * batch_size
                end = start + batch_size

                sess.run(train_step, feed_dict={
                    x: X_[start:end],
                    t: Y_[start:end]
                })
            loss_ = loss.eval(session=sess, feed_dict={
                x: X_train,
                t: Y_train
            })
            accuracy_ = accuracy.eval(session=sess, feed_dict={
                x: X_train,
                t: Y_train
            })
            self._history['loss'].append(loss_)
            self._history['accuracy'].append(accuracy_)

            if verbose:
                print('epoch:', epoch,
                      ' loss:', loss_,
                      ' accuracy:', accuracy_)

        return self._history

    def evaluate(self, X_test, Y_test):
        accuracy = self.accuracy(self._y, self._t)
        return accuracy.eval(session=self._sess, feed_dict={
            self._x: X_test,
            self._t: Y_test
        })

    def predict(self, X_test):
        prediction = self.prediction(self._y)
        return prediction.eval(session=self._sess, feed_dict={
            self._x: X_test
        })

3. Visualization policy

Boundary value visualization is performed by plotting the training results for each grid point. Consider the following black grid points as test data and predict the result.

If the blue line is the boundary value, ・ The result is "0" for the grid points above the blue line. ・ The result is "1" for the grid points below the blue line. You will get a result like this. This result is color-coded by the "contour line plot" described later.

4.meshgrid

Use numpy.meshgrid for grid points. For example, you can get a 5x5 list by doing the following:

x, y  = np.meshgrid(np.arange( 0, 10, 2),
                    np.arange( 1, 6, 1))
x
y

array([[0, 2, 4, 6, 8],
       [0, 2, 4, 6, 8],
       [0, 2, 4, 6, 8],
       [0, 2, 4, 6, 8],
       [0, 2, 4, 6, 8]])
array([[1, 1, 1, 1, 1],
       [2, 2, 2, 2, 2],
       [3, 3, 3, 3, 3],
       [4, 4, 4, 4, 4],
       [5, 5, 5, 5, 5]])

By specifying this on the x-axis and y-axis, the array of grid points will be as shown in the figure below. Starting from 0 in the grids, the grid points are in the order of the arrows.

grids = np.array([x.ravel(), y.ravel()]).T

5. Contour plot

Boundary values are drawn with the matplotlib contourf. Set the X-axis in the first argument, the Y-axis in the second argument, and the height in the third argument. The height is either "0" or "1" because it is the prediction result of binary classification in this case.

from matplotlib.colors import ListedColormap
cmap = ListedColormap( ( "mistyrose","lightcyan") )
plt.contourf(x, y, pred, cmap=cmap)

Assuming that the grid point prediction is pred, this code can draw a boundary line as shown in the figure below.

6. All chords

Below is the full code for the border plot, including training with the model.

import numpy as np
import tensorflow as tf
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
import matplotlib.pyplot as plt

np.random.seed(0)
tf.set_random_seed(0)

class ThreelayerPerceptron(object):
    '''
Initialization
    '''
    def __init__(self, n_in, n_hidden, n_out):
        self.n_in = n_in
        self.n_hidden = n_hidden
        self.n_out = n_out
        self.weights = []
        self.biases = []

        self._x = None
        self._y = None
        self._t = None,
        self._sess = None
        self._history = {
            'accuracy': [],
            'loss': []
        }

    '''
weight
    '''
    def weight_variable(self, shape):
        initial = tf.truncated_normal(shape, stddev=0.01)
        return tf.Variable(initial)

    '''
bias
    '''
    def bias_variable(self, shape):
        initial = tf.zeros(shape)
        return tf.Variable(initial)

    '''
Model definition (3-layer perceptron)
    '''
    def inference(self, x):
        
        #Input layer-Hidden layer
        self.weights.append(self.weight_variable([self.n_in, self.n_hidden]))
        self.biases.append(self.bias_variable([self.n_hidden]))
        h = tf.nn.sigmoid(tf.matmul(x, self.weights[-1]) + self.biases[-1])

        #Hidden layer-Output layer
        self.weights.append(self.weight_variable([self.n_hidden, self.n_out]))
        self.biases.append(self.bias_variable([self.n_out]))
        y = tf.nn.sigmoid(tf.matmul(h, self.weights[-1]) + self.biases[-1])

        return y

    '''
Loss function
    '''
    def loss(self, y, t):
        cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)))
        return cross_entropy

    '''
Optimization algorithm
    '''
    def training(self, loss):
        optimizer = tf.train.GradientDescentOptimizer(0.05)
        train_step = optimizer.minimize(loss)
        return train_step

    '''
Correct answer rate
    '''
    def accuracy(self, y, t):
        correct_prediction = tf.equal(tf.cast(tf.greater(y, 0.5),tf.float32), t)
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        return accuracy
    
    '''
Forecast
    '''
    def prediction(self, y):
        return tf.cast(tf.greater(y, 0.5),tf.float32)

    '''
Learning
    '''
    def fit(self, X_train, Y_train,
            nb_epoch=100, batch_size=100, p_keep=0.5,
            verbose=1):
        x = tf.placeholder(tf.float32, shape=[None, self.n_in])
        t = tf.placeholder(tf.float32, shape=[None, self.n_out])

        self._x = x
        self._t = t

        y = self.inference(x)
        loss = self.loss(y, t)
        train_step = self.training(loss)
        accuracy = self.accuracy(y, t)

        init = tf.global_variables_initializer()
        sess = tf.Session()
        sess.run(init)

        self._y = y
        self._sess = sess

        N_train = len(X_train)
        n_batches = N_train // batch_size

        for epoch in range(nb_epoch):
            X_, Y_ = shuffle(X_train, Y_train)

            for i in range(n_batches):
                start = i * batch_size
                end = start + batch_size

                sess.run(train_step, feed_dict={
                    x: X_[start:end],
                    t: Y_[start:end]
                })
            loss_ = loss.eval(session=sess, feed_dict={
                x: X_train,
                t: Y_train
            })
            accuracy_ = accuracy.eval(session=sess, feed_dict={
                x: X_train,
                t: Y_train
            })
            self._history['loss'].append(loss_)
            self._history['accuracy'].append(accuracy_)

            if verbose:
                print('epoch:', epoch,
                      ' loss:', loss_,
                      ' accuracy:', accuracy_)

        return self._history

    def evaluate(self, X_test, Y_test):
        accuracy = self.accuracy(self._y, self._t)
        return accuracy.eval(session=self._sess, feed_dict={
            self._x: X_test,
            self._t: Y_test
        })

    def predict(self, X_test):
        prediction = self.prediction(self._y)
        return prediction.eval(session=self._sess, feed_dict={
            self._x: X_test
        })


'''
Generate crescent data
'''
N = 300  #Total number of data
X, y = datasets.make_moons(N, noise=0.2)
Y = y.reshape(N, 1)

#Divide into training data and test data
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8)

'''
Data plot
'''
plt.figure(figsize=(10, 6))
plt.ylim([-1,1.5])
plt.xlim([-1.5,2.5])
plt.scatter(X[y==1][:,0], X[y==1][:,1], s=10, marker='.')
plt.scatter(X[y==0][:,0], X[y==0][:,1], s=10, marker='^')
plt.show()

'''
Model settings
'''
model = ThreelayerPerceptron(n_in=len(X[0]),
                             n_hidden=3,
                             n_out=len(Y[0]))

'''
Model learning
'''
history = model.fit(X_train, Y_train,
                    nb_epoch=400,
                    batch_size=20,
                    verbose=1)

'''
Evaluation of prediction accuracy
'''
accuracy = model.evaluate(X_test, Y_test)
print('accuracy: ', accuracy)

'''
Generate grid data to color code the graph
'''
meshgrids  = np.meshgrid(
               np.arange( -1.5, 2.6, 0.01 ),
               np.arange( -1, 1.6, 0.01 )
             )
xx = meshgrids[0]
yy = meshgrids[1]
#Make a list of grid points
grids = np.array([xx.ravel(), yy.ravel()]).T

'''
Get the prediction result of grid data
'''
pred = model.predict(grids)
pred = pred.reshape( xx.shape )

'''
Data plot
'''
from matplotlib.colors import ListedColormap
w_[1,1] * v_[1])
plt.figure(figsize=(10, 6))
plt.ylim([-1,1.5])
plt.xlim([-1.5,2.5])
cmap = ListedColormap( ( "mistyrose","lightcyan") )
plt.contourf(xx, yy, pred, cmap=cmap)
plt.scatter(X[y==1][:,0], X[y==1][:,1], s=10, marker='.')
plt.scatter(X[y==0][:,0], X[y==0][:,1], s=10, marker='^')
plt.show()

History

2020/03/08 First edition released 2020/03/09 Corrected definition of loss function

Visualize the boundary values of the multi-layer perceptron