# -*- coding: utf-8 -*-
"""
Functions to create network diagrams from a list of Layers.
References:
# Adapted from
https://github.com/ebenolson/Lasagne/blob/master/examples/draw_net.py
# TODO:
https://github.com/dnouri/nolearn/blob/master/nolearn/lasagne/visualize.py
"""
from __future__ import absolute_import, division, print_function
from operator import itemgetter
from os.path import join, exists
import numpy as np
import utool as ut
from wbia_cnn import utils
print, rrr, profile = ut.inject2(__name__)
[docs]def imwrite_theano_symbolic_graph(thean_expr):
import theano
graph_dpath = '.'
graph_fname = 'symbolic_graph.png'
graph_fpath = ut.unixjoin(graph_dpath, graph_fname)
ut.ensuredir(graph_dpath)
theano.printing.pydotprint(thean_expr, outfile=graph_fpath, var_with_name_simple=True)
ut.startfile(graph_fpath)
return graph_fpath
[docs]def draw_neural_net(ax, left, right, bottom, top, layer_sizes):
"""
References:
# Taken from
https://gist.github.com/craffel/2d727968c3aaebd10359
Draw a neural network cartoon using matplotilb.
Example:
>>> fig = plt.figure(figsize=(12, 12))
>>> draw_neural_net(fig.gca(), .1, .9, .1, .9, [4, 7, 2])
:parameters:
- ax : matplotlib.axes.AxesSubplot
The axes on which to plot the cartoon (get e.g. by plt.gca())
- left : float
The center of the leftmost node(s) will be placed here
- right : float
The center of the rightmost node(s) will be placed here
- bottom : float
The center of the bottommost node(s) will be placed here
- top : float
The center of the topmost node(s) will be placed here
- layer_sizes : list of int
List of layer sizes, including input and output dimensionality
"""
import matplotlib.pyplot as plt
# n_layers = len(layer_sizes)
v_spacing = (top - bottom) / float(max(layer_sizes))
h_spacing = (right - left) / float(len(layer_sizes) - 1)
# Nodes
for n, layer_size in enumerate(layer_sizes):
layer_top = v_spacing * (layer_size - 1) / 2.0 + (top + bottom) / 2.0
for m in range(layer_size):
circle = plt.Circle(
(n * h_spacing + left, layer_top - m * v_spacing),
v_spacing / 4.0,
color='w',
ec='k',
zorder=4,
)
ax.add_artist(circle)
# Edges
for n, (layer_size_a, layer_size_b) in enumerate(
zip(layer_sizes[:-1], layer_sizes[1:])
):
layer_top_a = v_spacing * (layer_size_a - 1) / 2.0 + (top + bottom) / 2.0
layer_top_b = v_spacing * (layer_size_b - 1) / 2.0 + (top + bottom) / 2.0
for m in range(layer_size_a):
for o in range(layer_size_b):
line = plt.Line2D(
[n * h_spacing + left, (n + 1) * h_spacing + left],
[layer_top_a - m * v_spacing, layer_top_b - o * v_spacing],
c='k',
)
ax.add_artist(line)
[docs]def show_arch_nx_graph(layers, fnum=None, fullinfo=True):
r"""
CommandLine:
python -m wbia_cnn.draw_net show_arch_nx_graph:0 --show
python -m wbia_cnn.draw_net show_arch_nx_graph:1 --show
Example0:
>>> # ENABLE_DOCTEST
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn import models
>>> model = models.mnist.MNISTModel(batch_size=128, output_dims=10,
>>> data_shape=(24, 24, 3))
>>> model.init_arch()
>>> layers = model.get_all_layers()
>>> show_arch_nx_graph(layers)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> ut.show_if_requested()
Example1:
>>> # ENABLE_DOCTEST
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn import models
>>> model = models.SiameseCenterSurroundModel(autoinit=True)
>>> layers = model.get_all_layers()
>>> show_arch_nx_graph(layers)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> ut.show_if_requested()
"""
import networkx as nx
import plottool as pt
from Lasagne import lasagne
# from matplotlib import offsetbox
# import matplotlib as mpl
REMOVE_BATCH_SIZE = True
from wbia_cnn import net_strs
def get_hex_color(layer_type):
if 'Input' in layer_type:
return '#A2CECE'
if 'Conv2D' in layer_type:
return '#7C9ABB'
if 'Dense' in layer_type:
return '#6CCF8D'
if 'Pool' in layer_type:
return '#9D9DD2'
if 'SoftMax' in layer_type:
return '#7E9FD9'
else:
return '#{0:x}'.format(hash(layer_type + 'salt') % 2 ** 24)
node_dict = {}
edge_list = []
edge_attrs = ut.ddict(dict)
# Make layer ids (ensure no duplicates)
layer_to_id = {
layer: repr(layer) if layer.name is None else layer.name for layer in set(layers)
}
keys_ = layer_to_id.keys()
dups = ut.find_duplicate_items(layer_to_id.values())
for dupval, dupidxs in dups.items():
newval_fmt = dupval + '_%d'
for layer in ut.take(keys_, dupidxs):
newid = ut.get_nonconflicting_string(newval_fmt, layer_to_id.values())
layer_to_id[layer] = newid
def layerid(layer):
return layer_to_id[layer]
main_nodes = []
for i, layer in enumerate(layers):
layer_info = net_strs.get_layer_info(layer)
layer_type = layer_info['classalias']
key = layerid(layer)
color = get_hex_color(layer_info['classalias'])
# Make label
lines = []
if layer_info['name'] is not None:
lines.append(layer_info['name'])
if fullinfo:
lines.append(layer_info['classalias'])
for attr, val in layer_info['layer_attrs'].items():
if attr == 'shape' and REMOVE_BATCH_SIZE:
val = val[1:]
if attr == 'output_shape' and REMOVE_BATCH_SIZE:
val = val[1:]
lines.append('{0}: {1}'.format(attr, val))
nonlinearity = layer_info.get('nonlinearity')
if nonlinearity is not None:
alias_map = {
'LeakyRectify': 'LReLU',
}
val = layer_info['nonlinearity']['type']
val = alias_map.get(val, val)
lines.append('nonlinearity:\n{0}'.format(val))
label = '\n'.join(lines)
# append node
is_main_layer = len(layer.params) > 0
# is_main_layer = len(lasagne.layers.get_all_params(layer, trainable=True)) > 0
if layer_info['classname'] in lasagne.layers.normalization.__all__:
is_main_layer = False
if layer_info['classname'] in lasagne.layers.special.__all__:
is_main_layer = False
if layer_info['classname'].startswith('BatchNorm'):
is_main_layer = False
if layer_info['classname'].startswith('ElemwiseSum'):
is_main_layer = True
if layer_type == 'Input':
is_main_layer = True
if hasattr(layer, '_is_main_layer'):
is_main_layer = layer._is_main_layer
# if getattr(layer, 'name', '') is not None and getattr(layer, 'name', '') .endswith('/sum'):
# is_main_layer = True
node_attr = dict(
name=key,
label=label,
color=color,
fillcolor=color,
style='filled',
is_main_layer=is_main_layer,
)
node_attr['is_main_layer'] = is_main_layer
if is_main_layer:
main_nodes.append(key)
node_attr['classalias'] = layer_info['classalias']
if is_main_layer or node_attr['classalias'].startswith('Conv'):
if hasattr(layer, 'shape'):
if len(layer.shape) == 3:
node_attr['out_size'] = (layer.shape[2], layer.shape[1])
node_attr['depth'] = layer.output_shape[0]
if hasattr(layer, 'output_shape'):
if len(layer.output_shape) == 4:
depth = layer.output_shape[1]
width, height = (layer.output_shape[3], layer.output_shape[2])
xshift = -width * (0.1 / (depth ** (1 / 3))) / 3
yshift = height * (0.1 / (depth ** (1 / 3))) / 2
node_attr['depth'] = depth
node_attr['xshift'] = xshift
node_attr['yshift'] = yshift
node_attr['out_size'] = (width, height)
if len(layer.output_shape) == 2:
node_attr['out_size'] = (1, layer.output_shape[1])
node_dict[key] = node_attr
_input_layers = []
if hasattr(layer, 'input_layers'):
_input_layers += layer.input_layers
if hasattr(layer, 'input_layer'):
_input_layers += [layer.input_layer]
for input_layer in _input_layers:
parent_key = layerid(input_layer)
edge = (parent_key, key)
edge_list.append(edge)
main_size_ = np.array((100, 100)) * 4
sub_size = np.array((75, 50)) * 4
# Setup scaled width and heights
out_size_list = [v['out_size'] for v in node_dict.values() if 'out_size' in v]
out_size_list = np.array(out_size_list)
# out_size_list = out_size_list[out_size_list.T[0] > 1]
area_arr = np.prod(out_size_list, axis=1)
main_outsize = np.array(out_size_list[area_arr.argmax()])
# main_outsize = np.array(out_size_list[area_arr.argmin()])
scale = main_size_ / main_outsize
scale_dense_max = 0.25
scale_dense_min = 8
for k, v in node_dict.items():
if v['is_main_layer'] or v['classalias'].startswith('Conv'):
if 'out_size' in v:
# Make dense layers more visible
if v['classalias'] == 'Dense':
v['shape'] = 'rect'
v['width'] = scale_dense_min
if v['out_size'][1] > main_outsize[1]:
v['height'] = v['out_size'][1] * scale[1] * scale_dense_max
elif v['out_size'][1] < scale_dense_min:
v['height'] = scale_dense_min * v['out_size'][1]
else:
v['height'] = v['out_size'][1]
elif v['classalias'].startswith('Conv'):
v['shape'] = 'stack'
# v['shape'] = 'rect'
v['width'] = v['out_size'][0] * scale[0]
v['height'] = v['out_size'][1] * scale[1]
else:
v['shape'] = 'rect'
v['width'] = v['out_size'][0] * scale[0]
v['height'] = v['out_size'][1] * scale[1]
else:
v['shape'] = 'rect'
v['width'] = main_size_[0]
v['height'] = main_size_[1]
else:
# v['shape'] = 'ellipse'
v['shape'] = 'rect'
v['style'] = 'rounded'
v['width'] = sub_size[0]
v['height'] = sub_size[1]
key_order = ut.take(layer_to_id, layers)
node_dict = ut.dict_subset(node_dict, key_order)
# print('node_dict = ' + ut.repr3(node_dict))
# Create the networkx graph structure
G = nx.DiGraph()
G.add_nodes_from(node_dict.items())
G.add_edges_from(edge_list)
for key, val in edge_attrs.items():
nx.set_edge_attributes(G, key, val)
# Add invisible structure
# main_nodes = [key for key, val in
# nx.get_node_attributes(G, 'is_main_layer').items() if val]
main_children = ut.odict()
# for n1, n2 in ut.itertwo(main_nodes):
# print('n1, n2 = %r %r' % (n1, n2))
# import utool
# utool.embed()
# children = ut.nx_all_nodes_between(G, n1, n2)
# if n1 in children:
# children.remove(n1)
# if n2 in children:
# children.remove(n2)
# main_children[n1] = children
# #pass
# main_children[main_nodes[-1]] = []
for n1 in main_nodes:
main_children[n1] = []
# Main nodes only place constraints on nodes in the next main group.
# Not their own
next_main = None
G.node[n1]['group'] = n1
for (_, n2) in nx.bfs_edges(G, n1):
if next_main is None:
if n2 in main_nodes:
next_main = n2
else:
G.node[n2]['group'] = n1
main_children[n1].append(n2)
else:
if n2 not in list(nx.descendants(G, next_main)):
G.node[n2]['group'] = n1
main_children[n1].append(n2)
# Custom positioning
x = 0
y = 1000
# print('main_children = %s' % (ut.repr3(main_children),))
# main_nodes = ut.isect(list(nx.topological_sort(G)), main_nodes)
xpad = main_size_[0] * 0.3
ypad = main_size_[1] * 0.3
# Draw each main node, and then put its children under it
# Then move to the left and draw the next main node.
cumwidth = 0
for n1 in main_nodes:
cumheight = 0
maxwidth = G.node[n1]['width']
for n2 in main_children[n1]:
maxwidth = max(maxwidth, G.node[n2]['width'])
cumwidth += xpad
cumwidth += maxwidth / 2
pos = np.array([x + cumwidth, y - cumheight])
G.node[n1]['pos'] = pos
G.node[n1]['pin'] = 'true'
height = G.node[n1]['height']
cumheight += height / 2
for n2 in main_children[n1]:
height = G.node[n2]['height']
cumheight += ypad
cumheight += height / 2
pos = np.array([x + cumwidth, y - cumheight])
G.node[n2]['pos'] = pos
G.node[n2]['pin'] = 'true'
cumheight += height / 2
cumwidth += maxwidth / 2
# Pin everybody
nx.set_node_attributes(G, 'pin', 'true')
layoutkw = dict(prog='neato', splines='line')
# layoutkw = dict(prog='neato', splines='spline')
layoutkw = dict(prog='neato', splines='ortho')
G_ = G.copy()
# delete lables for positioning
_labels = nx.get_node_attributes(G_, 'label')
ut.nx_delete_node_attr(G_, 'label')
nx.set_node_attributes(G_, 'label', '')
nolayout = False
if nolayout:
G_.remove_edges_from(list(G_.edges()))
else:
layout_info = pt.nx_agraph_layout(G_, inplace=True, **layoutkw) # NOQA
# reset labels
if not nolayout:
nx.set_node_attributes(G_, 'label', _labels)
_ = pt.show_nx(G_, fontsize=8, arrow_width=0.3, layout='custom', fnum=fnum) # NOQA
# pt.adjust_subplots(top=1, bot=0, left=0, right=1)
pt.plt.tight_layout()
[docs]def pydot_to_image(pydot_graph):
"""
References:
http://stackoverflow.com/questions/4596962/display-graph-without-saving-using-pydot
"""
from PIL import Image
from six.moves import StringIO
# from cStringIO import StringIO
png_str = pydot_graph.create_png(prog='dot')
sio = StringIO()
sio.write(png_str)
sio.seek(0)
pil_img = Image.open(sio)
img = np.asarray(pil_img.convert('RGB'))
img = img[..., ::-1] # to bgr
pil_img.close()
sio.close()
return img
# def make_architecture_image(layers, **kwargs):
# """
# Args:
# layers (list): List of the layers, as obtained from lasagne.layers.get_all_layers
# Kwargs:
# see docstring of make_architecture_pydot_graph for other options
# References:
# http://stackoverflow.com/questions/4596962/display-graph-without-saving-using-pydot
# CommandLine:
# python -m wbia_cnn.draw_net --test-make_architecture_image --show
# Example:
# >>> # ENABLE_DOCTEST
# >>> from wbia_cnn.draw_net import * # NOQA
# >>> from wbia_cnn import models
# >>> model = models.SiameseCenterSurroundModel(autoinit=True)
# >>> #model = models.DummyModel(autoinit=True)
# >>> layers = model.get_all_layers()
# >>> # execute function
# >>> kwargs = {}
# >>> img = make_architecture_image(layers, **kwargs)
# >>> print(img.shape)
# >>> ut.quit_if_noshow()
# >>> import plottool as pt
# >>> pt.imshow(img)
# >>> ut.show_if_requested()
# """
# # from IPython.display import Image # needed to render in notebook
# pydot_graph = make_architecture_pydot_graph(layers, **kwargs)
# img = pydot_to_image(pydot_graph)
# return img
# def imwrite_arch(layers, fpath, **kwargs):
# """
# Draws a network diagram to a file
# Args:
# layers (list): List of the layers, as obtained from lasagne.layers.get_all_layers
# fpath (str): The fpath to save output to.
# Kwargs:
# see docstring of make_architecture_pydot_graph for other options
# CommandLine:
# python -m wbia_cnn.draw_net --test-imwrite_arch --show
# Example:
# >>> # ENABLE_DOCTEST
# >>> from wbia_cnn.draw_net import * # NOQA
# >>> from wbia_cnn import models
# >>> #model = models.DummyModel(autoinit=True)
# >>> model = models.SiameseCenterSurroundModel(autoinit=True)
# >>> layers = model.get_all_layers()
# >>> fpath = ut.unixjoin(ut.ensure_app_resource_dir('wbia_cnn'), 'tmp.png')
# >>> # execute function
# >>> imwrite_arch(layers, fpath)
# >>> ut.quit_if_noshow()
# >>> ut.startfile(fpath)
# """
# pydot_graph = make_architecture_pydot_graph(layers, **kwargs)
# ext = fpath[fpath.rfind('.') + 1:]
# with open(fpath, 'w') as fid:
# fid.write(pydot_graph.create(format=ext))
[docs]def occlusion_heatmap(net, x, target, square_length=7):
"""An occlusion test that checks an image for its critical parts.
In this function, a square part of the image is occluded (i.e. set
to 0) and then the net is tested for its propensity to predict the
correct label. One should expect that this propensity shrinks of
critical parts of the image are occluded. If not, this indicates
overfitting.
Depending on the depth of the net and the size of the image, this
function may take awhile to finish, since one prediction for each
pixel of the image is made.
Currently, all color channels are occluded at the same time. Also,
this does not really work if images are randomly distorted by the
batch iterator.
See paper: Zeiler, Fergus 2013
Parameters
----------
net : NeuralNet instance
The neural net to test.
x : np.array
The input data, should be of shape (1, c, x, y). Only makes
sense with image data.
target : int
The true value of the image. If the net makes several
predictions, say 10 classes, this indicates which one to look
at.
square_length : int (default=7)
The length of the side of the square that occludes the image.
Must be an odd number.
Results
-------
heat_array : np.array (with same size as image)
An 2D np.array that at each point (i, j) contains the predicted
probability of the correct class if the image is occluded by a
square with center (i, j).
"""
from Lasagne.lasagne.layers import get_output_shape
if (x.ndim != 4) or x.shape[0] != 1:
raise ValueError(
'This function requires the input data to be of '
'shape (1, c, x, y), instead got {}'.format(x.shape)
)
if square_length % 2 == 0:
raise ValueError(
'Square length has to be an odd number, instead '
'got {}.'.format(square_length)
)
num_classes = get_output_shape(net.layers_[-1])[1]
img = x[0].copy()
bs, col, s0, s1 = x.shape
heat_array = np.zeros((s0, s1))
pad = square_length // 2 + 1
x_occluded = np.zeros((s1, col, s0, s1), dtype=img.dtype)
probs = np.zeros((s0, s1, num_classes))
# generate occluded images
for i in range(s0):
# batch s1 occluded images for faster prediction
for j in range(s1):
x_pad = np.pad(img, ((0, 0), (pad, pad), (pad, pad)), 'constant')
x_pad[:, i : i + square_length, j : j + square_length] = 0.0
x_occluded[j] = x_pad[:, pad:-pad, pad:-pad]
y_proba = net.predict_proba_Xb(x_occluded)
probs[i] = y_proba.reshape(s1, num_classes)
# from predicted probabilities, pick only those of target class
for i in range(s0):
for j in range(s1):
heat_array[i, j] = probs[i, j, target]
return heat_array
[docs]def _plot_heat_map(net, Xb, figsize, get_heat_image):
import plottool as pt
if Xb.ndim != 4:
raise ValueError(
'This function requires the input data to be of '
'shape (b, c, x, y), instead got {}'.format(Xb.shape)
)
num_images = Xb.shape[0]
if figsize[1] is None:
figsize = (figsize[0], num_images * figsize[0] / 3)
figs, axes = pt.plt.subplots(num_images, 3, figsize=figsize)
for ax in axes.flatten():
ax.set_xticks([])
ax.set_yticks([])
ax.axis('off')
for n in range(num_images):
heat_img = get_heat_image(net, Xb[n : n + 1, :, :, :], n)
ax = axes if num_images == 1 else axes[n]
img = Xb[n, :, :, :].mean(0)
ax[0].imshow(-img, interpolation='nearest', cmap='gray')
ax[0].set_title('image')
ax[1].imshow(-heat_img, interpolation='nearest', cmap='Reds')
ax[1].set_title('critical parts')
ax[2].imshow(-img, interpolation='nearest', cmap='gray')
ax[2].imshow(-heat_img, interpolation='nearest', cmap='Reds', alpha=0.6)
ax[2].set_title('super-imposed')
return pt.plt
[docs]def plot_occlusion(net, Xb, target, square_length=7, figsize=(9, None)):
"""Plot which parts of an image are particularly import for the
net to classify the image correctly.
See paper: Zeiler, Fergus 2013
Parameters
----------
net : NeuralNet instance
The neural net to test.
Xb : numpy.array
The input data, should be of shape (b, c, 0, 1). Only makes
sense with image data.
target : list or numpy.array of ints
The true values of the image. If the net makes several
predictions, say 10 classes, this indicates which one to look
at. If more than one sample is passed to Xb, each of them needs
its own target.
square_length : int (default=7)
The length of the side of the square that occludes the image.
Must be an odd number.
figsize : tuple (int, int)
Size of the figure.
Plots
-----
Figure with 3 subplots: the original image, the occlusion heatmap,
and both images super-imposed.
"""
return _plot_heat_map(
net,
Xb,
figsize,
lambda net, Xb, n: occlusion_heatmap(net, Xb, target[n], square_length),
)
[docs]def plot_saliency(net, Xb, figsize=(9, None)):
def saliency_map(input, output, pred, Xb):
import theano.tensor as T
from Lasagne.lasagne.objectives import binary_crossentropy
score = -binary_crossentropy(output[:, pred], np.array([1])).sum()
heat_map_ = np.abs(T.grad(score, input).eval({input: Xb}))
return heat_map_
def saliency_map_net(net, Xb):
from Lasagne.lasagne.layers import get_output
input = net.layers_[0].input_var
output = get_output(net.layers_[-1])
pred = output.eval({input: Xb}).argmax(axis=1)
heat_map_ = saliency_map(input, output, pred, Xb)
heat_img = heat_map_[0].transpose(1, 2, 0).squeeze()
return heat_img
return _plot_heat_map(net, Xb, figsize, lambda net, Xb, n: -saliency_map_net(net, Xb))
[docs]class Dream(object):
"""
https://groups.google.com/forum/#!topic/lasagne-users/UxZpNthZfq0
http://arxiv.org/pdf/1312.6034.pdf
http://igva2012.wikispaces.asu.edu/file/view/Erhan+2009+Visualizing+higher+layer+features+of+a+deep+network.pdf
#TODO
https://arxiv.org/pdf/1605.09304v3.pdf
Class model visualization. Sort of like a deep-dream
CommandLine:
python -m wbia_cnn.draw_net Dream --show
Example:
>>> # DISABLE_DOCTEST
>>> # Assumes mnist is trained
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn.models import mnist
>>> model, dataset = mnist.testdata_mnist(dropout=.5)
>>> model.init_arch()
>>> model.load_model_state()
>>> target_labels = 3
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> #pt.qt4ensure()
>>> dream = Dream(model, niters=200)
>>> img = dream.make_class_images(target_labels)
>>> pt.imshow(img)
>>> ut.show_if_requested()
"""
[docs] def saliency(dream, Xb, yb):
"""
num = 10
Xb = model.prepare_data(X_test[0:num])
yb = y_test[0:num]
dpath = ''
dataset = None
"""
dpath = '.'
import theano.tensor as T
from Lasagne import lasagne
import vtool as vt
import theano
model = dream.model
# Use current weights to find the score of a particular class
Xb_shared = theano.shared(Xb)
yb_shared = theano.shared(yb.astype(np.int32))
# Get the final layer and remove the softmax nonlinearity to access the
# pre-activation. (Softmax encourages minimization of other classes)
import copy
# softmax = copy.copy(model.output_layer)
# softmax.nonlinearity = lasagne.nonlinearities.identity
softmax = copy.copy(model.output_layer)
class_probs = lasagne.layers.get_output(softmax, Xb_shared, deterministic=True)
# werid way to index into position of target
flat_idx = (T.arange(yb_shared.shape[0]) * class_probs.shape[1]) + yb_shared
class_probs_target = T.flatten(class_probs)[flat_idx]
# Get derivative of scores for the target class wrt the input
d_score_wrt_input = theano.grad(class_probs_target.mean(), Xb_shared)
w = np.array(d_score_wrt_input.eval())
saliency = w.max(axis=1, keepdims=True)
outs = saliency.transpose((0, 2, 3, 1))
X = Xb.transpose((0, 2, 3, 1))
for count in range(len(Xb)):
img = X[count]
y = yb[count]
out = vt.norm01(outs[count])
overlay = vt.blend_images_multiply(out, img)
vt.imwrite(
join(dpath, 'out%d_A_image_t=%s.jpg' % (count, y)),
vt.rectify_to_uint8(img),
)
vt.imwrite(
join(dpath, 'out%d_B_heat_t=%s.jpg' % (count, y)),
vt.rectify_to_uint8(out),
)
vt.imwrite(
join(dpath, 'out%d_C_overlay_t=%s.jpg' % (count, y)),
vt.rectify_to_uint8(overlay),
)
def __init__(
dream, model, init='gauss', niters=100, update_rate=1e-2, weight_decay=1e-5
):
dream.model = model
dream.init = init
dream.niters = niters
dream.update_rate = update_rate
dream.weight_decay = weight_decay
# FIXME: cached vars assumes not much changes
dream.shared_images = None
dream.step_fn = None
[docs] def make_class_images(dream, target_labels):
import theano
from theano import tensor as T # NOQA
import utool as ut
was_scalar = not ut.isiterable(target_labels)
target_labels = ut.ensure_iterable(target_labels)
if True:
# We are forcing a batch size for this visualization
input_shape = (len(target_labels),) + dream.model.input_shape[1:]
else:
# Maybe some cnn layers cant take variable batches?
input_shape = dream.model.input_shape
b, c, w, h = input_shape
assert len(target_labels) <= b, 'batch size too small'
initial_state = dream._make_init_state()
# make image a shared variable that you can update
if dream.shared_images is None:
dream.shared_images = theano.shared(initial_state)
else:
dream.shared_images.set_value(initial_state)
if dream.step_fn is None:
dream.step_fn = dream._make_objective(dream.shared_images, target_labels)
# Optimize objective via backpropogation for a few iterations
for _ in ut.ProgIter(range(dream.niters), lbl='making class model img', bs=True):
dream.step_fn()
# print('objective = %r' % (objective,))
out = dream._postprocess_class_image(
dream.shared_images, target_labels, was_scalar
)
return out
[docs] def _make_init_state(dream):
r"""
CommandLine:
python -m wbia_cnn.draw_net _make_init_state --show
Example:
>>> # DISABLE_DOCTEST
>>> # Assumes mnist is trained
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn.models import mnist
>>> model, dataset = mnist.testdata_mnist(dropout=.5)
>>> model.init_arch()
>>> dream = Dream(model, init='rgauss', niters=200)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> import vtool as vt
>>> #pt.qt4ensure()
>>> initial_state = dream._make_init_state().transpose((0, 2, 3, 1))[0]
>>> pt.imshow(initial_state, pnum=(1, 2, 1), fnum=1)
>>> pt.imshow(vt.norm01(initial_state), pnum=(1, 2, 2), fnum=1)
>>> ut.show_if_requested()
"""
init = dream.init
input_shape = dream.model.input_shape
b, c, w, h = input_shape
rng = np.random.RandomState(0)
if init is None or init == 'zeros':
# intializing to zeros seems to do nothing on mnist data
initial_state = np.zeros(input_shape, dtype=np.float32)
if init in ['rand', 'random']:
initial_state = rng.rand(*input_shape)
elif init in ['randn']:
initial_state = np.abs(rng.randn(*input_shape)) / 6
initial_state = np.clip(initial_state, 0, 1)
elif init in ['gauss']:
import vtool as vt
initial_state = np.array(
[[vt.gaussian_patch((h, w), sigma=None) for _ in range(c)]] * b
)
# initial_state /= initial_state.max()
elif init in ['rgauss']:
import vtool as vt
initial_state = np.array(
[[vt.gaussian_patch((h, w), sigma=None) for _ in range(c)]] * b
)
# initial_state /= initial_state.max()
raug = np.abs(rng.randn(*input_shape)) * (initial_state.max() / 12)
initial_state += raug
initial_state = np.clip(initial_state, 0, 1)
elif init in ['perlin']:
import vtool as vt
b, c, w, h = input_shape
initial_state = np.array(
[[vt.perlin_noise((w, h), rng=rng) for _ in range(c)]] * b
)
initial_state = initial_state.astype(np.float32) / 255
initial_state = initial_state.astype(np.float32)
return initial_state
[docs] def _make_objective(dream, shared_images, target_labels):
r"""
The goal is to optimize
S_c = score of class c before softmax nonlinearity
argmax_{I} S_c(I) - \lambda \elltwo{I}
max(S_c(I) - lambda * norm(I, 2))
"""
from Lasagne import lasagne
import copy
import theano
from theano import tensor as T # NOQA
print('Making dream objective')
# Get the final layer and remove the softmax nonlinearity to access the
# pre-activation. (Softmax encourages minimization of other classes)
softmax = copy.copy(dream.model.output_layer)
softmax.nonlinearity = lasagne.nonlinearities.identity
# Overwrite lasagne's InputLayer with the image
# Build expression to represent class scores wrt the image
class_scores = lasagne.layers.get_output(
softmax, shared_images, deterministic=True
)
# Get the class score that represents our class of interest
# simultaniously generate as many classes as were requested.
max_term_batch = [class_scores[bx, y] for bx, y in enumerate(target_labels)]
max_term = T.mean(max_term_batch)
# Get the squared L2 norm of the image values
flat_img = T.reshape(
shared_images, (shared_images.shape[0], T.prod(shared_images.shape[1:]))
)
reg_term_batch = (flat_img ** 2).sum(axis=1)
reg_term = T.mean(reg_term_batch)
objective = max_term - dream.weight_decay * reg_term
# Compute the gradient of the maximization objective
# with respect to the image
grads = theano.grad(objective, wrt=shared_images)
# compile a function that does this update
step_fn = theano.function(
inputs=[],
# outputs could be empty, but returning objective allows us to
# monitor progress
outputs=[objective],
updates={shared_images: shared_images + dream.update_rate * grads},
)
return step_fn
[docs] def _postprocess_class_image(dream, shared_images, target_labels, was_scalar):
# return final state of the image
Xb = shared_images.get_value()
X = Xb.transpose((0, 2, 3, 1))
out_ = X[0 : len(target_labels)]
import vtool as vt # NOQA
out = out_.copy()
out = vt.norm01(out) * 255
out = np.round(out).astype(np.uint8)
if was_scalar:
out = out[0]
return out
[docs] def generate_class_images(dream, target_labels):
"""
import plottool as pt
fnum = None
kw = dict(init='gauss', niters=500, update_rate=.05, weight_decay=1e-4)
target_labels = list(range(model.output_dims))
dream = draw_net.Dream(model, **kw)
target_labels = 8
images = list(dream.generate_class_images(target_labels))
vid = vt.make_video(images, 'dynimg.pimj', fps=1, is_color=False, format='PIM1')
vid = vt.make_video2(images, 'dynimg')
import matplotlib.pyplot as plt
ims = []
for img in imgs:
im = plt.imshow(img[:, :, 0], interpolation='nearest', cmap='gray')
ims.append([im])
import matplotlib.animation as animation
fig = plt.figure()
ani = animation.ArtistAnimation(fig, ims, interval=50, blit=True,
repeat_delay=1000)
ani.save('dynamic_images.mp4')
ut.startfile('dynamic_images.mp4')
plt.show()
"""
import theano
from theano import tensor as T # NOQA
import utool as ut
input_shape = dream.model.input_shape
b, c, w, h = input_shape
was_scalar = not ut.isiterable(target_labels)
target_labels = ut.ensure_iterable(target_labels)
assert len(target_labels) <= b, 'batch size too small'
initial_state = dream._make_init_state()
shared_images = theano.shared(initial_state.astype(np.float32))
step_fn = dream._make_objective(shared_images, target_labels)
out = dream._postprocess_class_image(shared_images, target_labels, was_scalar)
yield out
for _ in ut.ProgIter(range(dream.niters), lbl='class dream', bs=True):
step_fn()
# objective = step_fn()
# print('objective = %r' % (objective,))
out = dream._postprocess_class_image(shared_images, target_labels, was_scalar)
yield out
[docs]def show_saliency_heatmap(model, dataset):
"""
https://github.com/dnouri/nolearn/blob/master/nolearn/lasagne/visualize.py
Example:
>>> # DISABLE_DOCTEST
>>> # Assumes mnist is trained
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn import ingest_data
>>> from wbia_cnn.models import MNISTModel
>>> dataset = ingest_data.grab_mnist_category_dataset()
>>> model = MNISTModel(batch_size=128, data_shape=dataset.data_shape,
>>> name='bnorm',
>>> output_dims=len(dataset.unique_labels),
>>> batch_norm=True,
>>> dataset_dpath=dataset.dataset_dpath)
>>> model.encoder = None
>>> model.init_arch()
>>> model.load_model_state()
>>> import plottool as pt
>>> pt.qt4ensure()
>>> show_saliency_heatmap(model, dataset)
>>> ut.show_if_requested()
"""
if dataset.has_subset('valid'):
X_train, y_train = dataset.subset('train')
_, _, X_valid, y_valid = model._prefit(X_train, y_train)
else:
X_valid, y_valid = dataset.subset('valid')
net = model
num = 4
start = 0
X = X_valid[start : start + num]
y = y_valid[start : start + num]
Xb = net.prepare_input(X)
plot_occlusion(net, Xb, y)
# plot_saliency(net, Xb)
[docs]def show_convolutional_weights(
all_weights, use_color=None, limit=144, fnum=None, pnum=(1, 1, 1)
):
r"""
Args:
all_weights (?):
use_color (bool):
limit (int):
CommandLine:
python -m wbia_cnn.draw_net --test-show_convolutional_weights --show
python -m wbia_cnn.draw_net --test-show_convolutional_weights --show --index=1
# Need to fit mnist first
python -m wbia_cnn _ModelFitting.fit:1 --vd
Example:
>>> # DISABLE_DOCTEST
>>> # Assumes mnist is trained
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn import ingest_data
>>> from wbia_cnn.models import MNISTModel
>>> dataset = ingest_data.grab_mnist_category_dataset()
>>> model = MNISTModel(batch_size=128, data_shape=dataset.data_shape,
>>> name='bnorm',
>>> output_dims=len(dataset.unique_labels),
>>> batch_norm=True,
>>> dataset_dpath=dataset.dataset_dpath)
>>> model.encoder = None
>>> model.init_arch()
>>> model.load_model_state()
>>> nn_layers = model.get_all_layers()
>>> weighted_layers = [layer for layer in nn_layers if hasattr(layer, 'W')]
>>> all_weights = weighted_layers[0].W.get_value()
>>> print('all_weights.shape = %r' % (all_weights.shape,))
>>> use_color = None
>>> limit = 64
>>> fig = show_convolutional_weights(all_weights, use_color, limit)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> pt.qt4ensure()
>>> fig = show_convolutional_weights(all_weights, use_color, limit)
>>> ut.show_if_requested()
Example:
>>> # ENABLE_DOCTEST
>>> from wbia_cnn.draw_net import * # NOQA
>>> from wbia_cnn import models
>>> from Lasagne.lasagne import layers
>>> model = models.SiameseCenterSurroundModel(autoinit=True)
>>> output_layer = model.get_output_layer()
>>> nn_layers = layers.get_all_layers(output_layer)
>>> weighted_layers = [layer for layer in nn_layers if hasattr(layer, 'W')]
>>> index = ut.get_argval('--index', type_=int, default=0)
>>> all_weights = weighted_layers[index].W.get_value()
>>> print('all_weights.shape = %r' % (all_weights.shape,))
>>> use_color = None
>>> limit = 64
>>> fig = show_convolutional_weights(all_weights, use_color, limit)
>>> ut.show_if_requested()
"""
import plottool as pt
if fnum is None:
fnum = pt.next_fnum()
fig = pt.figure(fnum=fnum, pnum=pnum, docla=True)
num, channels, height, width = all_weights.shape
if use_color is None:
# Try to infer if use_color should be shown
use_color = channels == 3
stacked_img = make_conv_weight_image(all_weights, limit)
# ax = fig.add_subplot(111)
if len(stacked_img.shape) == 3 and stacked_img.shape[-1] == 1:
stacked_img = stacked_img.reshape(stacked_img.shape[:-1])
pt.imshow(stacked_img)
return fig
[docs]def make_conv_weight_image(all_weights, limit=144):
""" just makes the image ndarray of the weights """
import vtool as vt
import cv2
# Try to infer if use_color should be shown
num, channels, height, width = all_weights.shape
# Try to infer if use_color should be shown
use_color = channels == 3
# non-use_color features need to be flattened
if not use_color:
all_weights_ = all_weights.reshape(num * channels, height, width, 1)
else:
# convert from theano to cv2 BGR
all_weights_ = utils.convert_theano_images_to_cv2_images(all_weights)
# convert from BGR to RGB
all_weights_ = all_weights_[..., ::-1]
# cv2.cvtColor(all_weights_[-1], cv2.COLOR_BGR2RGB)
# Limit all_weights_
# num = all_weights_.shape[0]
num, height, width, channels = all_weights_.shape
if limit is not None and num > limit:
all_weights_ = all_weights_[:limit]
num = all_weights_.shape[0]
# Convert weight values to image values
normalize_individually = False
if normalize_individually:
# Normalize each feature individually
all_max = vt.multiaxis_reduce(np.amax, all_weights_, startaxis=1)
all_min = vt.multiaxis_reduce(np.amin, all_weights_, startaxis=1)
all_domain = all_max - all_min
extra_dims = (None,) * (len(all_weights_.shape) - 1)
broadcaster = (slice(None),) + extra_dims
all_features = (
(all_weights_ - all_min[broadcaster]) * (255.0 / all_domain[broadcaster])
).astype(np.uint8)
else:
# Normalize jointly across all filters
_max = all_weights_.max()
_min = all_weights_.min()
_domain = _max - _min
all_features = ((all_weights_ - _min) * (255.0 / _domain)).astype(np.uint8)
# import scipy.misc
# resize feature, give them a border, and stack them together
new_height, new_width = max(32, height), max(32, width)
nbp_ = 1 # num border pixels
_resized_features = np.array(
[
cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_NEAREST)
for img in all_features
]
)
resized_features = _resized_features.reshape(num, new_height, new_width, channels)
border_shape = (num, new_height + (nbp_ * 2), new_width + (nbp_ * 2), channels)
bordered_features = np.zeros(border_shape, dtype=resized_features.dtype)
bordered_features[:, nbp_:-nbp_, nbp_:-nbp_, :] = resized_features
# img_list = bordered_features
stacked_img = vt.stack_square_images(bordered_features)
return stacked_img
[docs]def output_confusion_matrix(X_test, results_path, test_results, model, **kwargs):
""" currently hacky implementation, fix it later """
loss, accu_test, prob_list, auglbl_list, pred_list, conf_list = test_results
# Output confusion matrix
mapping_fn = None
if model is not None:
mapping_fn = getattr(model, 'label_order_mapping', None)
# TODO: THIS NEEDS TO BE FIXED
label_list = list(range(kwargs.get('output_dims')))
# Encode labels if avaialble
# encoder = kwargs.get('encoder', None)
encoder = getattr(model, 'encoder', None)
if encoder is not None:
label_list = encoder.inverse_transform(label_list)
# Make confusion matrix (pass X to write out failed cases)
show_confusion_matrix(
auglbl_list, pred_list, label_list, results_path, mapping_fn, X_test
)
[docs]def save_confusion_matrix(
results_path, correct_y, predict_y, category_list, mapping_fn=None, data_x=None
):
import plottool as pt
fig = show_confusion_matrix(
correct_y, predict_y, category_list, mapping_fn=mapping_fn, data_x=data_x
)
output_fpath = join(results_path, 'confusion.png')
pt.save_figure(fig, fpath=output_fpath)
return output_fpath
[docs]def show_confusion_matrix(
correct_y, predict_y, category_list, results_path, mapping_fn=None, data_x=None
):
"""
Given the correct and predict labels, show the confusion matrix
Args:
correct_y (list of int): the list of correct labels
predict_y (list of int): the list of predict assigned labels
category_list (list of str): the category list of all categories
Displays:
matplotlib: graph of the confusion matrix
Returns:
None
TODO FIXME and simplify
"""
import matplotlib.pyplot as plt
import cv2
confused_examples = join(results_path, 'confused')
if data_x is not None:
if exists(confused_examples):
ut.remove_dirs(confused_examples, quiet=True)
ut.ensuredir(confused_examples)
size = len(category_list)
if mapping_fn is None:
# Identity
category_mapping = {key: index for index, key in enumerate(category_list)}
category_list_ = category_list
else:
category_mapping = mapping_fn(category_list)
assert all(
[category in category_mapping.keys() for category in category_list]
), 'Not all categories are mapped'
values = list(category_mapping.values())
assert len(list(set(values))) == len(
values
), 'Mapped categories have a duplicate assignment'
assert 0 in values, 'Mapped categories must have a 0 index'
temp = list(category_mapping.iteritems())
temp = sorted(temp, key=itemgetter(1))
category_list_ = [t[0] for t in temp]
confidences = np.zeros((size, size))
counters = {}
for index, (correct, predict) in enumerate(zip(correct_y, predict_y)):
# Ensure type
correct = int(correct)
predict = int(predict)
# Get the "text" label
example_correct_label = category_list[correct]
example_predict_label = category_list[predict]
# Perform any mapping that needs to be done
correct_ = category_mapping[example_correct_label]
predict_ = category_mapping[example_predict_label]
# Add to the confidence matrix
confidences[correct_][predict_] += 1
if data_x is not None and correct_ != predict_:
example = data_x[index]
example_name = '%s^SEEN_INCORRECTLY_AS^%s' % (
example_correct_label,
example_predict_label,
)
if example_name not in counters.keys():
counters[example_name] = 0
counter = counters[example_name]
counters[example_name] += 1
example_name = '%s^%d.png' % (example_name, counter)
example_path = join(confused_examples, example_name)
# TODO: make write confused examples function
cv2.imwrite(example_path, example)
row_sums = np.sum(confidences, axis=1)
norm_conf = (confidences.T / row_sums).T
fig = plt.figure(1)
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect(1)
res = ax.imshow(np.array(norm_conf), cmap=plt.cm.jet, interpolation='nearest')
for x in range(size):
for y in range(size):
ax.annotate(
str(int(confidences[x][y])),
xy=(y, x),
horizontalalignment='center',
verticalalignment='center',
)
cb = fig.colorbar(res) # NOQA
plt.xticks(np.arange(size), category_list_[0:size], rotation=90)
plt.yticks(np.arange(size), category_list_[0:size])
margin_small = 0.1
margin_large = 0.9
plt.subplots_adjust(
left=margin_small, right=margin_large, bottom=margin_small, top=margin_large
)
plt.xlabel('Predicted')
plt.ylabel('Correct')
return fig
if __name__ == '__main__':
"""
CommandLine:
python -m wbia_cnn.draw_net
python -m wbia_cnn.draw_net --allexamples
python -m wbia_cnn.draw_net --allexamples --noface --nosrc
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()