Commit f0b2c8ddf401443a1a15e8d27dea29e06868079f

Authored by Cédric Traizet
1 parent b2fc854c
Exists in master and in 1 other branch dev

refactoring of the ae model seems to work, denoising ae is not implemented yet

include/AutoencoderModel.h
... ... @@ -86,8 +86,8 @@ public:
86 86 template <class T>
87 87 void TrainOneLayer(shark::AbstractStoppingCriterion<T> & criterion,unsigned int, unsigned int,double, double, shark::Data<shark::RealVector> &, std::ostream&);
88 88  
89   - template <class T>
90   - void TrainOneSparseLayer(shark::AbstractStoppingCriterion<T> & criterion,unsigned int, unsigned int,double, double,double, shark::Data<shark::RealVector> &, std::ostream&);
  89 + template <class T, class Autoencoder>
  90 + void TrainOneSparseLayer(shark::AbstractStoppingCriterion<T> & criterion,Autoencoder &, unsigned int, unsigned int,double, double,double, shark::Data<shark::RealVector> &, std::ostream&);
91 91  
92 92 protected:
93 93 AutoencoderModel();
... ... @@ -102,8 +102,6 @@ private:
102 102 /** Network attributes */
103 103 //std::vector<AutoencoderType> m_net;
104 104 NetworkType m_net;
105   -
106   -
107 105 itk::Array<unsigned int> m_NumberOfHiddenNeurons;
108 106 /** Training parameters */
109 107 unsigned int m_NumberOfIterations; // stop the training after a fixed number of iterations
... ...
include/AutoencoderModel.txx
... ... @@ -48,12 +48,64 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::Train()
48 48 ofs << "learning curve" << std::endl;
49 49 }
50 50  
  51 + /// Initialization of the feed forward neural network
  52 + std::vector<size_t> layers;
  53 + layers.push_back(shark::dataDimension(inputSamples));
  54 + for (unsigned int i = 0 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
  55 + {
  56 + layers.push_back(m_NumberOfHiddenNeurons[i]);
  57 + std::cout << m_NumberOfHiddenNeurons.Size() << std::endl;
  58 + }
  59 + // another loop for the decoder should be added, for now i just add the output layer size
  60 + std::cout << "i?" << static_cast<int>(m_NumberOfHiddenNeurons.Size()-1) << std::endl;
  61 + for (unsigned int i = std::max(0,static_cast<int>(m_NumberOfHiddenNeurons.Size()-1)) ; i > 0; --i)
  62 + {
  63 + std::cout << i << std::endl;
  64 + layers.push_back(m_NumberOfHiddenNeurons[i-1]);
  65 + }
51 66  
  67 + layers.push_back(shark::dataDimension(inputSamples));
  68 + m_net.setStructure(layers);
  69 + shark::initRandomNormal(m_net,0.1);
  70 +
  71 +
  72 + /// Training of the first Autoencoder (first and last layer of the FF network)
  73 + if (m_Epsilon > 0){
  74 + shark::TrainingProgress<> criterion(5,m_Epsilon);
  75 +
  76 + if (m_Noise[0] != 0) // Shark doesn't allow to train a layer using a sparsity term AND a noisy input. (shark::SparseAutoencoderError takes an autoen
  77 + {
  78 + TrainOneLayer(criterion,0 , m_NumberOfHiddenNeurons[0],m_Noise[0],m_Regularization[0], inputSamples,ofs);
  79 + }
  80 + else
  81 + {
  82 + OutAutoencoderType net;
  83 + TrainOneSparseLayer( criterion, net , 0 , m_NumberOfHiddenNeurons[0],m_Rho[0],m_Beta[0],m_Regularization[0],inputSamples, ofs);
  84 + }
  85 + criterion.reset();
  86 + }
52 87  
  88 + else {
  89 + shark::MaxIterations<> criterion(m_NumberOfIterations);
  90 +
  91 + if (m_Noise[0] != 0) // Shark doesn't allow to train a layer using a sparsity term AND a noisy input. (shark::SparseAutoencoderError takes an autoen
  92 + {
  93 + TrainOneLayer(criterion,0, m_NumberOfHiddenNeurons[0],m_Noise[0],m_Regularization[0], inputSamples, ofs);
  94 + }
  95 + else
  96 + {
  97 + OutAutoencoderType net;
  98 + TrainOneSparseLayer(criterion, net, 0, m_NumberOfHiddenNeurons[0],m_Rho[0],m_Beta[0],m_Regularization[0], inputSamples, ofs);
  99 + }
  100 + criterion.reset();
  101 + }
  102 +
  103 +
  104 + /// Training of the other autoencoders
53 105 if (m_Epsilon > 0){
54 106 shark::TrainingProgress<> criterion(5,m_Epsilon);
55 107  
56   - for (unsigned int i = 0 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
  108 + for (unsigned int i = 1 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
57 109 {
58 110 if (m_Noise[i] != 0) // Shark doesn't allow to train a layer using a sparsity term AND a noisy input. (shark::SparseAutoencoderError takes an autoen
59 111 {
... ... @@ -61,7 +113,8 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::Train()
61 113 }
62 114 else
63 115 {
64   - TrainOneSparseLayer( criterion,i , m_NumberOfHiddenNeurons[i],m_Rho[i],m_Beta[i],m_Regularization[i],inputSamples, ofs);
  116 + AutoencoderType net;
  117 + TrainOneSparseLayer( criterion, net , i , m_NumberOfHiddenNeurons[i],m_Rho[i],m_Beta[i],m_Regularization[i],inputSamples, ofs);
65 118 }
66 119 criterion.reset();
67 120 }
... ... @@ -71,7 +124,7 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::Train()
71 124 else {
72 125 shark::MaxIterations<> criterion(m_NumberOfIterations);
73 126  
74   - for (unsigned int i = 0 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
  127 + for (unsigned int i = 1 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
75 128 {
76 129 if (m_Noise[i] != 0) // Shark doesn't allow to train a layer using a sparsity term AND a noisy input. (shark::SparseAutoencoderError takes an autoen
77 130 {
... ... @@ -79,7 +132,8 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::Train()
79 132 }
80 133 else
81 134 {
82   - TrainOneSparseLayer(criterion,i, m_NumberOfHiddenNeurons[i],m_Rho[i],m_Beta[i],m_Regularization[i], inputSamples, ofs);
  135 + AutoencoderType net;
  136 + TrainOneSparseLayer(criterion, net, i, m_NumberOfHiddenNeurons[i],m_Rho[i],m_Beta[i],m_Regularization[i], inputSamples, ofs);
83 137 }
84 138 criterion.reset();
85 139 }
... ... @@ -95,8 +149,8 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::TrainOneLayer(shark::AbstractStop
95 149  
96 150 std::size_t inputs = dataDimension(samples);
97 151 net.setStructure(inputs, nbneuron);
98   - //initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
99   - initRandomUniform(net,-1,1);
  152 + initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
  153 + //initRandomUniform(net,-1,1);
100 154 shark::ImpulseNoiseModel noise(noise_strength,0.0); //set an input pixel with probability m_Noise to 0
101 155 shark::ConcatenatedModel<shark::RealVector,shark::RealVector> model = noise>> net;
102 156 shark::LabeledData<shark::RealVector,shark::RealVector> trainSet(samples,samples);//labels identical to inputs
... ... @@ -136,15 +190,15 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::TrainOneLayer(shark::AbstractStop
136 190  
137 191  
138 192 template <class TInputValue, class NeuronType>
139   -template <class T>
140   -void AutoencoderModel<TInputValue,NeuronType>::TrainOneSparseLayer(shark::AbstractStoppingCriterion<T> & criterion,unsigned int layer_index, unsigned int nbneuron,double rho,double beta, double regularization, shark::Data<shark::RealVector> &samples, std::ostream& File)
  193 +template <class T, class Autoencoder>
  194 +void AutoencoderModel<TInputValue,NeuronType>::TrainOneSparseLayer(shark::AbstractStoppingCriterion<T> & criterion, Autoencoder & net, unsigned int layer_index, unsigned int nbneuron,double rho,double beta, double regularization, shark::Data<shark::RealVector> &samples, std::ostream& File)
141 195 {
142   - AutoencoderType net;
  196 + //AutoencoderType net;
143 197  
144 198 std::size_t inputs = dataDimension(samples);
145 199 net.setStructure(inputs, nbneuron);
146   - //initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
147   - initRandomUniform(net,-1,1);
  200 + initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
  201 + //initRandomUniform(net,-1,1);
148 202 shark::LabeledData<shark::RealVector,shark::RealVector> trainSet(samples,samples);//labels identical to inputs
149 203 shark::SquaredLoss<shark::RealVector> loss;
150 204 shark::SparseAutoencoderError error(trainSet,&net, &loss, rho, beta);
... ... @@ -172,9 +226,12 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::TrainOneSparseLayer(shark::Abstra
172 226 }
173 227 net.setParameterVector(optimizer.solution().point);
174 228 //m_net.push_back(net);
175   - m_net.setLayer(layer_index,net.encoderMatrix(),net.hiddenBias());
176   - m_net.setLayer( m_NumberOfHiddenNeurons.Size()*2 - 1 - layer_index,net.decoderMatrix(),net.outputBias());
  229 + m_net.setLayer(layer_index,net.encoderMatrix(),net.hiddenBias()); // Copy the encoder in the FF neural network
  230 + m_net.setLayer( m_NumberOfHiddenNeurons.Size()*2 - 1 - layer_index,net.decoderMatrix(),net.outputBias()); // Copy the decoder in the FF neural network
177 231 samples = net.encode(samples);
  232 + std::cout << "numero zero " << m_net.layerMatrix(layer_index) << "i " << layer_index <<std::endl <<std::endl;
  233 + std::cout << "numero uno " << m_net.layerMatrix(m_NumberOfHiddenNeurons.Size()*2 - 1 - layer_index) << "i " << m_NumberOfHiddenNeurons.Size()*2 - 1 - layer_index << std::endl <<std::endl;
  234 +
178 235 }
179 236  
180 237 template <class TInputValue, class NeuronType>
... ... @@ -202,6 +259,7 @@ bool AutoencoderModel&lt;TInputValue,NeuronType&gt;::CanWriteFile(const std::string &amp;
202 259 template <class TInputValue, class NeuronType>
203 260 void AutoencoderModel<TInputValue,NeuronType>::Save(const std::string & filename, const std::string & name)
204 261 {
  262 + std::cout << "saving model ..." << std::endl;
205 263 std::ofstream ofs(filename);
206 264 ofs << m_net.name() << std::endl; // the first line of the model file contains a key
207 265 boost::archive::polymorphic_text_oarchive oa(ofs);
... ... @@ -271,13 +329,23 @@ void AutoencoderModel&lt;TInputValue,NeuronType&gt;::Load(const std::string &amp; filename
271 329 //m_net.read(ia);
272 330 ia >> m_net;
273 331 ifs.close();
274   -
275   - m_NumberOfHiddenNeurons.SetSize(m_net.size());
276   - for (int i=0; i<m_net.size(); i++){
  332 +
  333 + // This gives us the dimension if we keep the encoder and decoder
  334 + size_t feature_layer_index = m_net.layerMatrices().size()/2;
  335 + this->m_Dimension = m_net.layerMatrix(feature_layer_index).size1(); // number of neurons in the feature layer (first dimension of the first decoder weight matrix)
  336 + std::cout << this->m_Dimension << std::endl;
  337 +
  338 +
  339 +
  340 + /* This might not be needed
  341 + int number_of_layers = m_net.layerMatrices().size()/2;
  342 + std::cout << "number of layers" << number_of_layers << std::endl;
  343 + m_NumberOfHiddenNeurons.SetSize(number_of_layers);
  344 + for (int i=0; i<number_of_layers; i++){
277 345 m_NumberOfHiddenNeurons[i] = m_net[i].numberOfHiddenNeurons();
278 346 }
279   - this->m_Dimension = m_NumberOfHiddenNeurons[m_net.size()-1];
280   -
  347 + this->m_Dimension = m_NumberOfHiddenNeurons[number_of_layers-1];
  348 + */
281 349 }
282 350  
283 351  
... ... @@ -285,7 +353,7 @@ template &lt;class TInputValue, class NeuronType&gt;
285 353 typename AutoencoderModel<TInputValue,NeuronType>::TargetSampleType
286 354 AutoencoderModel<TInputValue,NeuronType>::DoPredict(const InputSampleType & value, ConfidenceValueType * quality) const
287 355 {
288   - /*
  356 +
289 357 shark::RealVector samples(value.Size());
290 358 for(size_t i = 0; i < value.Size();i++)
291 359 {
... ... @@ -296,18 +364,20 @@ AutoencoderModel&lt;TInputValue,NeuronType&gt;::DoPredict(const InputSampleType &amp; valu
296 364 features.push_back(samples);
297 365  
298 366 shark::Data<shark::RealVector> data = shark::createDataFromRange(features);
299   -
  367 +
  368 + data = m_net.evalLayer( m_net.layerMatrices().size()/2 ,data); // features layer for a network containing the encoder and decoder part
  369 + /*
300 370 for (int i=0; i<m_net.size(); i++){ // loop over all autoencoders in m_net
301 371 data = m_net[i].encode(data);
302   - }
  372 + }*/
303 373 TargetSampleType target;
304   - target.SetSize(m_NumberOfHiddenNeurons[m_net.size()-1]);
  374 + target.SetSize(this->m_Dimension);
305 375  
306   - for(unsigned int a = 0; a < m_NumberOfHiddenNeurons[m_net.size()-1]; ++a){
  376 + for(unsigned int a = 0; a < this->m_Dimension; ++a){
307 377 target[a]=data.element(0)[a];
308 378 }
309 379 return target;
310   - */
  380 +
311 381 }
312 382  
313 383  
... ... @@ -315,27 +385,28 @@ template &lt;class TInputValue, class NeuronType&gt;
315 385 void AutoencoderModel<TInputValue,NeuronType>
316 386 ::DoPredictBatch(const InputListSampleType *input, const unsigned int & startIndex, const unsigned int & size, TargetListSampleType * targets, ConfidenceListSampleType * quality) const
317 387 {
318   - /*
  388 +
319 389 std::vector<shark::RealVector> features;
320 390 Shark::ListSampleRangeToSharkVector(input, features,startIndex,size);
321 391 shark::Data<shark::RealVector> data = shark::createDataFromRange(features);
322 392 TargetSampleType target;
323   -
  393 + /*
324 394 for (auto net :m_net ){ // loop over all autoencoders in m_net
325 395 data = net.encode(data);
326 396 }
327   -
  397 + */
  398 + data = m_net.evalLayer( m_net.layerMatrices().size()/2 ,data); // features layer for a network containing the encoder and decoder part
328 399 unsigned int id = startIndex;
329   - target.SetSize(m_NumberOfHiddenNeurons[m_net.size()-1]);
  400 + target.SetSize(this->m_Dimension);
330 401 for(const auto& p : data.elements())
331 402 {
332   - for(unsigned int a = 0; a < m_NumberOfHiddenNeurons[m_net.size()-1]; ++a){
  403 + for(unsigned int a = 0; a < this->m_Dimension; ++a){
333 404 target[a]=p[a];
334 405 }
335 406 targets->SetMeasurementVector(id,target);
336 407 ++id;
337 408 }
338   - */
  409 +
339 410 }
340 411  
341 412 } // namespace otb
... ...