Commit ae4a5b12 authored by Evan Shelhamer's avatar Evan Shelhamer
Browse files

Merge pull request #2505 from ronghanghu/matcaffe3

MatCaffe: overhaul and improve the MATLAB interface
parents b12c1710 d07e5f79
......@@ -65,7 +65,7 @@ NONGEN_CXX_SRCS := $(shell find \
src/$(PROJECT) \
include/$(PROJECT) \
python/$(PROJECT) \
matlab/$(PROJECT) \
matlab/+$(PROJECT)/private \
examples \
tools \
-name "*.cpp" -or -name "*.hpp" -or -name "*.cu" -or -name "*.cuh")
......@@ -79,12 +79,12 @@ NONEMPTY_LINT_REPORT := $(BUILD_DIR)/$(LINT_EXT)
PY$(PROJECT)_SRC := python/$(PROJECT)/_$(PROJECT).cpp
PY$(PROJECT)_SO := python/$(PROJECT)/_$(PROJECT).so
PY$(PROJECT)_HXX := include/$(PROJECT)/python_layer.hpp
# MAT$(PROJECT)_SRC is the matlab wrapper for $(PROJECT)
MAT$(PROJECT)_SRC := matlab/$(PROJECT)/mat$(PROJECT).cpp
# MAT$(PROJECT)_SRC is the mex entrance point of matlab package for $(PROJECT)
MAT$(PROJECT)_SRC := matlab/+$(PROJECT)/private/$(PROJECT)_.cpp
ifneq ($(MATLAB_DIR),)
MAT_SO_EXT := $(shell $(MATLAB_DIR)/bin/mexext)
endif
MAT$(PROJECT)_SO := matlab/$(PROJECT)/$(PROJECT).$(MAT_SO_EXT)
MAT$(PROJECT)_SO := matlab/+$(PROJECT)/private/$(PROJECT)_.$(MAT_SO_EXT)
##############################
# Derive generated files
......@@ -118,7 +118,7 @@ GTEST_OBJ := $(addprefix $(BUILD_DIR)/, ${GTEST_SRC:.cpp=.o})
EXAMPLE_OBJS := $(addprefix $(BUILD_DIR)/, ${EXAMPLE_SRCS:.cpp=.o})
# Output files for automatic dependency generation
DEPS := ${CXX_OBJS:.o=.d} ${CU_OBJS:.o=.d} ${TEST_CXX_OBJS:.o=.d} \
${TEST_CU_OBJS:.o=.d}
${TEST_CU_OBJS:.o=.d} $(BUILD_DIR)/${MAT$(PROJECT)_SO:.$(MAT_SO_EXT)=.d}
# tool, example, and test bins
TOOL_BINS := ${TOOL_OBJS:.o=.bin}
EXAMPLE_BINS := ${EXAMPLE_OBJS:.o=.bin}
......@@ -460,6 +460,9 @@ $(MAT$(PROJECT)_SO): $(MAT$(PROJECT)_SRC) $(STATIC_NAME)
CXX="$(CXX)" \
CXXFLAGS="\$$CXXFLAGS $(MATLAB_CXXFLAGS)" \
CXXLIBS="\$$CXXLIBS $(STATIC_LINK_COMMAND) $(LDFLAGS)" -output $@
@ if [ -f "$(PROJECT)_.d" ]; then \
mv -f $(PROJECT)_.d $(BUILD_DIR)/${MAT$(PROJECT)_SO:.$(MAT_SO_EXT)=.d}; \
fi
runtest: $(TEST_ALL_BIN)
$(TOOL_BUILD_DIR)/caffe
......@@ -467,6 +470,9 @@ runtest: $(TEST_ALL_BIN)
pytest: py
cd python; python -m unittest discover -s caffe/test
mattest: mat
cd matlab; $(MATLAB_DIR)/bin/matlab -nodisplay -r 'caffe.run_tests(), exit()'
warn: $(EMPTY_WARN_REPORT)
......
......@@ -67,10 +67,213 @@ Compile pycaffe by `make pycaffe`. The module dir caffe/python/caffe should be i
## MATLAB
The MATLAB interface -- matcaffe -- is the `caffe` mex and its helper m-files in caffe/matlab. Load models, do forward and backward, extract output and read-only model weights, and load the binaryproto format mean as a matrix.
The MATLAB interface -- matcaffe -- is the `caffe` package in caffe/matlab in which you can integrate Caffe in your Matlab code.
A MATLAB demo is in caffe/matlab/caffe/matcaffe_demo.m
In MatCaffe, you can
Note that MATLAB matrices and memory are in column-major layout counter to Caffe's row-major layout! Double-check your work accordingly.
* Creating multiple Nets in Matlab
* Do forward and backward computation
* Access any layer within a network, and any parameter blob in a layer
* Get and set data or diff to any blob within a network, not restricting to input blobs or output blobs
* Save a network's parameters to file, and load parameters from file
* Reshape a blob and reshape a network
* Edit network parameter and do network surgery
* Create multiple Solvers in Matlab for training
* Resume training from solver snapshots
* Access train net and test nets in a solver
* Run for a certain number of iterations and give back control to Matlab
* Intermingle arbitrary Matlab code with gradient steps
Compile matcaffe by `make matcaffe`.
An ILSVRC image classification demo is in caffe/matlab/demo/classification_demo.m (you need to download BVLC CaffeNet from [Model Zoo](http://caffe.berkeleyvision.org/model_zoo.html) to run it).
### Build MatCaffe
Build MatCaffe with `make all matcaffe`. After that, you may test it using `make mattest`.
Common issue: if you run into error messages like `libstdc++.so.6:version 'GLIBCXX_3.4.15' not found` during `make mattest`, then it usually means that your Matlab's runtime libraries do not match your compile-time libraries. You may need to do the following before you start Matlab:
export LD_LIBRARY_PATH=/opt/intel/mkl/lib/intel64:/usr/local/cuda/lib64
export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6
Or the equivalent based on where things are installed on your system, and do `make mattest` again to see if the issue is fixed. Note: this issue is sometimes more complicated since during its startup Matlab may overwrite your `LD_LIBRARY_PATH` environment variable. You can run `!ldd ./matlab/+caffe/private/caffe_.mexa64` (the mex extension may differ on your system) in Matlab to see its runtime libraries, and preload your compile-time libraries by exporting them to your `LD_PRELOAD` environment variable.
After successful building and testing, add this package to Matlab search PATH by starting `matlab` from caffe root folder and running the following commands in Matlab command window.
addpath ./matlab
You can save your Matlab search PATH by running `savepath` so that you don't have to run the command above again every time you use MatCaffe.
### Use MatCaffe
MatCaffe is very similar to PyCaffe in usage.
Examples below shows detailed usages and assumes you have downloaded BVLC CaffeNet from [Model Zoo](http://caffe.berkeleyvision.org/model_zoo.html) and started `matlab` from caffe root folder.
model = './models/bvlc_reference_caffenet/deploy.prototxt';
weights = './models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel';
#### Set mode and device
**Mode and device should always be set BEFORE you create a net or a solver.**
Use CPU:
caffe.set_mode_cpu();
Use GPU and specify its gpu_id:
caffe.set_mode_gpu();
caffe.set_device(gpu_id);
#### Create a network and access its layers and blobs
Create a network:
net = caffe.Net(model, weights, 'test'); % create net and load weights
Or
net = caffe.Net(model, 'test'); % create net but not load weights
net.copy_from(weights); % load weights
which creates `net` object as
Net with properties:
layer_vec: [1x23 caffe.Layer]
blob_vec: [1x15 caffe.Blob]
inputs: {'data'}
outputs: {'prob'}
name2layer_index: [23x1 containers.Map]
name2blob_index: [15x1 containers.Map]
layer_names: {23x1 cell}
blob_names: {15x1 cell}
The two `containers.Map` objects are useful to find the index of a layer or a blob by its name.
You have access to every blob in this network. To fill blob 'data' with all ones:
net.blobs('data').set_data(ones(net.blobs('data').shape));
To multiply all values in blob 'data' by 10:
net.blobs('data').set_data(net.blobs('data').get_data() * 10);
**Be aware that since Matlab is 1-indexed and column-major, the usual 4 blob dimensions in Matlab are `[width, height, channels, num]`, and `width` is the fastest dimension. Also be aware that images are in BGR channels.** Also, Caffe uses single-precision float data. If your data is not single, `set_data` will automatically convert it to single.
You also have access to every layer, so you can do network surgery. For example, to multiply conv1 parameters by 10:
net.params('conv1', 1).set_data(net.params('conv1', 1).get_data() * 10); % set weights
net.params('conv1', 2).set_data(net.params('conv1', 2).get_data() * 10); % set bias
Alternatively, you can use
net.layers('conv1').params(1).set_data(net.layers('conv1').params(1).get_data() * 10);
net.layers('conv1').params(2).set_data(net.layers('conv1').params(2).get_data() * 10);
To save the network you just modified:
net.save('my_net.caffemodel');
To get a layer's type (string):
layer_type = net.layers('conv1').type;
#### Forward and backward
Forward pass can be done using `net.forward` or `net.forward_prefilled`. Function `net.forward` takes in a cell array of N-D arrays containing data of input blob(s) and outputs a cell array containing data from output blob(s). Function `net.forward_prefilled` uses existing data in input blob(s) during forward pass, takes no input and produces no output. After creating some data for input blobs like `data = rand(net.blobs('data').shape);` you can run
res = net.forward({data});
prob = res{1};
Or
net.blobs('data').set_data(data);
net.forward_prefilled();
prob = net.blobs('prob').get_data();
Backward is similar using `net.backward` or `net.backward_prefilled` and replacing `get_data` and `set_data` with `get_diff` and `set_diff`. After creating some gradients for output blobs like `prob_diff = rand(net.blobs('prob').shape);` you can run
res = net.backward({prob_diff});
data_diff = res{1};
Or
net.blobs('prob').set_diff(prob_diff);
net.backward_prefilled();
data_diff = net.blobs('data').get_diff();
**However, the backward computation above doesn't get correct results, because Caffe decides that the network does not need backward computation. To get correct backward results, you need to set `'force_backward: true'` in your network prototxt.**
After performing forward or backward pass, you can also get the data or diff in internal blobs. For example, to extract pool5 features after forward pass:
pool5_feat = net.blobs('pool5').get_data();
#### Reshape
Assume you want to run 1 image at a time instead of 10:
net.blobs('data').reshape([227 227 3 1]); % reshape blob 'data'
net.reshape();
Then the whole network is reshaped, and now `net.blobs('prob').shape` should be `[1000 1]`;
#### Training
Assume you have created training and validation lmdbs following our [ImageNET Tutorial](http://caffe.berkeleyvision.org/gathered/examples/imagenet.html), to create a solver and train on ILSVRC 2012 classification dataset:
solver = caffe.Solver('./models/bvlc_reference_caffenet/solver.prototxt');
which creates `solver` object as
Solver with properties:
net: [1x1 caffe.Net]
test_nets: [1x1 caffe.Net]
To train:
solver.solve();
Or train for only 1000 iterations (so that you can do something to its net before training more iterations)
solver.step(1000);
To get iteration number:
iter = solver.iter();
To get its network:
train_net = solver.net;
test_net = solver.test_nets(1);
To resume from a snapshot "your_snapshot.solverstate":
solver.restore('your_snapshot.solverstate');
#### Input and output
`caffe.io` class provides basic input functions `load_image` and `read_mean`. For example, to read ILSVRC 2012 mean file (assume you have downloaded imagenet example auxiliary files by running `./data/ilsvrc12/get_ilsvrc_aux.sh`):
mean_data = caffe.io.read_mean('./data/ilsvrc12/imagenet_mean.binaryproto');
To read Caffe's example image and resize to `[width, height]` and suppose we want `width = 256; height = 256;`
im_data = caffe.io.load_image('./examples/images/cat.jpg');
im_data = imresize(im_data, [width, height]); % resize using Matlab's imresize
**Keep in mind that `width` is the fastest dimension and channels are BGR, which is different from the usual way that Matlab stores an image.** If you don't want to use `caffe.io.load_image` and prefer to load an image by yourself, you can do
im_data = imread('./examples/images/cat.jpg'); % read image
im_data = im_data(:, :, [3, 2, 1]); % convert from RGB to BGR
im_data = permute(im_data, [2, 1, 3]); % permute width and height
im_data = single(im_data); % convert to single precision
Also, you may take a look at caffe/matlab/demo/classification_demo.m to see how to prepare input by taking crops from an image.
We show in caffe/matlab/hdf5creation how to read and write HDF5 data with Matlab. We do not provide extra functions for data output as Matlab itself is already quite powerful in output.
#### Clear nets and solvers
Call `caffe.reset_all()` to clear all solvers and stand-alone nets you have created.
classdef test_net < matlab.unittest.TestCase
properties
num_output
model_file
net
end
methods (Static)
function model_file = simple_net_file(num_output)
model_file = tempname();
fid = fopen(model_file, 'w');
fprintf(fid, [ ...
'name: "testnet" force_backward: true\n' ...
'layer { type: "DummyData" name: "data" top: "data" top: "label"\n' ...
'dummy_data_param { num: 5 channels: 2 height: 3 width: 4\n' ...
' num: 5 channels: 1 height: 1 width: 1\n' ...
' data_filler { type: "gaussian" std: 1 }\n' ...
' data_filler { type: "constant" } } }\n' ...
'layer { type: "Convolution" name: "conv" bottom: "data" top: "conv"\n' ...
' convolution_param { num_output: 11 kernel_size: 2 pad: 3\n' ...
' weight_filler { type: "gaussian" std: 1 }\n' ...
' bias_filler { type: "constant" value: 2 } }\n' ...
' param { decay_mult: 1 } param { decay_mult: 0 }\n' ...
' }\n' ...
'layer { type: "InnerProduct" name: "ip" bottom: "conv" top: "ip"\n' ...
' inner_product_param { num_output: ' num2str(num_output) ...
' weight_filler { type: "gaussian" std: 2.5 }\n' ...
' bias_filler { type: "constant" value: -3 } } }\n' ...
'layer { type: "SoftmaxWithLoss" name: "loss" bottom: "ip" bottom: "label"\n' ...
' top: "loss" }' ]);
fclose(fid);
end
end
methods
function self = test_net()
self.num_output = 13;
self.model_file = caffe.test.test_net.simple_net_file(self.num_output);
self.net = caffe.Net(self.model_file, 'train');
% also make sure get_solver runs
caffe.get_net(self.model_file, 'train');
% fill in valid labels
self.net.blobs('label').set_data(randi( ...
self.num_output - 1, self.net.blobs('label').shape));
delete(self.model_file);
end
end
methods (Test)
function self = test_blob(self)
self.net.blobs('data').set_data(10 * ones(self.net.blobs('data').shape));
self.verifyEqual(self.net.blobs('data').get_data(), ...
10 * ones(self.net.blobs('data').shape, 'single'));
self.net.blobs('data').set_diff(-2 * ones(self.net.blobs('data').shape));
self.verifyEqual(self.net.blobs('data').get_diff(), ...
-2 * ones(self.net.blobs('data').shape, 'single'));
original_shape = self.net.blobs('data').shape;
self.net.blobs('data').reshape([6 5 4 3 2 1]);
self.verifyEqual(self.net.blobs('data').shape, [6 5 4 3 2 1]);
self.net.blobs('data').reshape(original_shape);
self.net.reshape();
end
function self = test_layer(self)
self.verifyEqual(self.net.params('conv', 1).shape, [2 2 2 11]);
self.verifyEqual(self.net.layers('conv').params(2).shape, 11);
self.verifyEqual(self.net.layers('conv').type(), 'Convolution');
end
function test_forward_backward(self)
self.net.forward_prefilled();
self.net.backward_prefilled();
end
function test_inputs_outputs(self)
self.verifyEqual(self.net.inputs, cell(0, 1))
self.verifyEqual(self.net.outputs, {'loss'});
end
function test_save_and_read(self)
weights_file = tempname();
self.net.save(weights_file);
model_file2 = caffe.test.test_net.simple_net_file(self.num_output);
net2 = caffe.Net(model_file2, 'train');
net2.copy_from(weights_file);
net3 = caffe.Net(model_file2, weights_file, 'train');
delete(model_file2);
delete(weights_file);
for l = 1:length(self.net.layer_vec)
for i = 1:length(self.net.layer_vec(l).params)
self.verifyEqual(self.net.layer_vec(l).params(i).get_data(), ...
net2.layer_vec(l).params(i).get_data());
self.verifyEqual(self.net.layer_vec(l).params(i).get_data(), ...
net3.layer_vec(l).params(i).get_data());
end
end
end
end
end
classdef test_solver < matlab.unittest.TestCase
properties
num_output
solver
end
methods
function self = test_solver()
self.num_output = 13;
model_file = caffe.test.test_net.simple_net_file(self.num_output);
solver_file = tempname();
fid = fopen(solver_file, 'w');
fprintf(fid, [ ...
'net: "' model_file '"\n' ...
'test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9\n' ...
'weight_decay: 0.0005 lr_policy: "inv" gamma: 0.0001 power: 0.75\n' ...
'display: 100 max_iter: 100 snapshot_after_train: false\n' ]);
fclose(fid);
self.solver = caffe.Solver(solver_file);
% also make sure get_solver runs
caffe.get_solver(solver_file);
caffe.set_mode_cpu();
% fill in valid labels
self.solver.net.blobs('label').set_data(randi( ...
self.num_output - 1, self.solver.net.blobs('label').shape));
self.solver.test_nets(1).blobs('label').set_data(randi( ...
self.num_output - 1, self.solver.test_nets(1).blobs('label').shape));
delete(solver_file);
delete(model_file);
end
end
methods (Test)
function test_solve(self)
self.verifyEqual(self.solver.iter(), 0)
self.solver.step(30);
self.verifyEqual(self.solver.iter(), 30)
self.solver.solve()
self.verifyEqual(self.solver.iter(), 100)
end
end
end
classdef Blob < handle
% Wrapper class of caffe::Blob in matlab
properties (Access = private)
hBlob_self
end
methods
function self = Blob(hBlob_blob)
CHECK(is_valid_handle(hBlob_blob), 'invalid Blob handle');
% setup self handle
self.hBlob_self = hBlob_blob;
end
function shape = shape(self)
shape = caffe_('blob_get_shape', self.hBlob_self);
end
function reshape(self, shape)
shape = self.check_and_preprocess_shape(shape);
caffe_('blob_reshape', self.hBlob_self, shape);
end
function data = get_data(self)
data = caffe_('blob_get_data', self.hBlob_self);
end
function set_data(self, data)
data = self.check_and_preprocess_data(data);
caffe_('blob_set_data', self.hBlob_self, data);
end
function diff = get_diff(self)
diff = caffe_('blob_get_diff', self.hBlob_self);
end
function set_diff(self, diff)
diff = self.check_and_preprocess_data(diff);
caffe_('blob_set_diff', self.hBlob_self, diff);
end
end
methods (Access = private)
function shape = check_and_preprocess_shape(~, shape)
CHECK(isempty(shape) || (isnumeric(shape) && isrow(shape)), ...
'shape must be a integer row vector');
shape = double(shape);
end
function data = check_and_preprocess_data(self, data)
CHECK(isnumeric(data), 'data or diff must be numeric types');
self.check_data_size_matches(data);
if ~isa(data, 'single')
data = single(data);
end
end
function check_data_size_matches(self, data)
% check whether size of data matches shape of this blob
% note: matlab arrays always have at least 2 dimensions. To compare
% shape between size of data and shape of this blob, extend shape of
% this blob to have at least 2 dimensions
self_shape_extended = self.shape;
if isempty(self_shape_extended)
% target blob is a scalar (0 dim)
self_shape_extended = [1, 1];
elseif isscalar(self_shape_extended)
% target blob is a vector (1 dim)
self_shape_extended = [self_shape_extended, 1];
end
% Also, matlab cannot have tailing dimension 1 for ndim > 2, so you
% cannot create 20 x 10 x 1 x 1 array in matlab as it becomes 20 x 10
% Extend matlab arrays to have tailing dimension 1 during shape match
data_size_extended = ...
[size(data), ones(1, length(self_shape_extended) - ndims(data))];
is_matched = ...
(length(self_shape_extended) == length(data_size_extended)) ...
&& all(self_shape_extended == data_size_extended);
CHECK(is_matched, ...
sprintf('%s, input data/diff size: [ %s] vs target blob shape: [ %s]', ...
'input data/diff size does not match target blob shape', ...
sprintf('%d ', data_size_extended), sprintf('%d ', self_shape_extended)));
end
end
end
classdef Layer < handle
% Wrapper class of caffe::Layer in matlab
properties (Access = private)
hLayer_self
attributes
% attributes fields:
% hBlob_blobs
end
properties (SetAccess = private)
params
end
methods
function self = Layer(hLayer_layer)
CHECK(is_valid_handle(hLayer_layer), 'invalid Layer handle');
% setup self handle and attributes
self.hLayer_self = hLayer_layer;
self.attributes = caffe_('layer_get_attr', self.hLayer_self);
% setup weights
self.params = caffe.Blob.empty();
for n = 1:length(self.attributes.hBlob_blobs)
self.params(n) = caffe.Blob(self.attributes.hBlob_blobs(n));
end
end
function layer_type = type(self)
layer_type = caffe_('layer_get_type', self.hLayer_self);
end
end
end
classdef Net < handle
% Wrapper class of caffe::Net in matlab
properties (Access = private)
hNet_self
attributes
% attribute fields
% hLayer_layers
% hBlob_blobs
% input_blob_indices
% output_blob_indices
% layer_names
% blob_names
end
properties (SetAccess = private)
layer_vec
blob_vec
inputs
outputs
name2layer_index
name2blob_index
layer_names
blob_names
end
methods
function self = Net(varargin)
% decide whether to construct a net from model_file or handle
if ~(nargin == 1 && isstruct(varargin{1}))
% construct a net from model_file
self = caffe.get_net(varargin{:});
return
end
% construct a net from handle
hNet_net = varargin{1};
CHECK(is_valid_handle(hNet_net), 'invalid Net handle');
% setup self handle and attributes
self.hNet_self = hNet_net;
self.attributes = caffe_('net_get_attr', self.hNet_self);
% setup layer_vec
self.layer_vec = caffe.Layer.empty();
for n = 1:length(self.attributes.hLayer_layers)
self.layer_vec(n) = caffe.Layer(self.attributes.hLayer_layers(n));
end
% setup blob_vec
self.blob_vec = caffe.Blob.empty();
for n = 1:length(self.attributes.hBlob_blobs);
self.blob_vec(n) = caffe.Blob(self.attributes.hBlob_blobs(n));
end
% setup input and output blob and their names
% note: add 1 to indices as matlab is 1-indexed while C++ is 0-indexed
self.inputs = ...
self.attributes.blob_names(self.attributes.input_blob_indices + 1);
self.outputs = ...
self.attributes.blob_names(self.attributes.output_blob_indices + 1);
% create map objects to map from name to layers and blobs
self.name2layer_index = containers.Map(self.attributes.layer_names, ...
1:length(self.attributes.layer_names));
self.name2blob_index = containers.Map(self.attributes.blob_names, ...
1:length(self.attributes.blob_names));
% expose layer_names and blob_names for public read access
self.layer_names = self.attributes.layer_names;
self.blob_names = self.attributes.blob_names;
end
function layer = layers(self, layer_name)
CHECK(ischar(layer_name), 'layer_name must be a string');
layer = self.layer_vec(self.name2layer_index(layer_name));
end
function blob = blobs(self, blob_name)
CHECK(ischar(blob_name), 'blob_name must be a string');
blob = self.blob_vec(self.name2blob_index(blob_name));
end
function blob = params(self, layer_name, blob_index)
CHECK(ischar(layer_name), 'layer_name must be a string');
CHECK(isscalar(blob_index), 'blob_index must be a scalar');
blob = self.layer_vec(self.name2layer_index(layer_name)).params(blob_index);
end
function forward_prefilled(self)
caffe_('net_forward', self.hNet_self);
end
function backward_prefilled(self)
caffe_('net_backward', self.hNet_self);
end
function res = forward(self, input_data)
CHECK(iscell(input_data), 'input_data must be a cell array');
CHECK(length(input_data) == length(self.inputs), ...
'input data cell length must match input blob number');