Commit a79f0d47ca65cf0905615b7d9dbeea5b515062e8

Authored by Cédric Traizet
1 parent 00fb23c7

Sparse ae added, parameters can now be defined for each layers (noise, regularization, rho and beta)

app/cbDimensionalityReductionVector.cxx
@@ -29,6 +29,7 @@ @@ -29,6 +29,7 @@
29 #include "DimensionalityReductionModelFactory.h" 29 #include "DimensionalityReductionModelFactory.h"
30 #include "DimensionalityReductionModel.h" 30 #include "DimensionalityReductionModel.h"
31 #include <time.h> 31 #include <time.h>
  32 +
32 namespace otb 33 namespace otb
33 { 34 {
34 namespace Wrapper 35 namespace Wrapper
@@ -115,7 +116,7 @@ class CbDimensionalityReductionVector : public Application @@ -115,7 +116,7 @@ class CbDimensionalityReductionVector : public Application
115 SetDocExampleParameterValue("featout", "perimeter area width"); 116 SetDocExampleParameterValue("featout", "perimeter area width");
116 //SetOfficialDocLink(); 117 //SetOfficialDocLink();
117 } 118 }
118 - 119 + //
119 void DoUpdateParameters() ITK_OVERRIDE 120 void DoUpdateParameters() ITK_OVERRIDE
120 { 121 {
121 122
@@ -271,7 +272,7 @@ class CbDimensionalityReductionVector : public Application @@ -271,7 +272,7 @@ class CbDimensionalityReductionVector : public Application
271 272
272 otb::ogr::Layer outLayer = output->GetLayer(0); 273 otb::ogr::Layer outLayer = output->GetLayer(0);
273 OGRErr errStart = outLayer.ogr().StartTransaction(); 274 OGRErr errStart = outLayer.ogr().StartTransaction();
274 - /* 275 +
275 if (errStart != OGRERR_NONE) 276 if (errStart != OGRERR_NONE)
276 { 277 {
277 itkExceptionMacro(<< "Unable to start transaction for OGR layer " << outLayer.ogr().GetName() << "."); 278 itkExceptionMacro(<< "Unable to start transaction for OGR layer " << outLayer.ogr().GetName() << ".");
@@ -279,25 +280,28 @@ class CbDimensionalityReductionVector : public Application @@ -279,25 +280,28 @@ class CbDimensionalityReductionVector : public Application
279 280
280 // Add the field of prediction in the output layer if field not exist 281 // Add the field of prediction in the output layer if field not exist
281 282
282 - OGRFeatureDefn &layerDefn = layer.GetLayerDefn();  
283 - int idx = layerDefn.GetFieldIndex(GetParameterStringList("featout").c_str());  
284 -  
285 - if (idx >= 0) 283 + for (int i=0; i<GetParameterStringList("featout").size() ;i++)
286 { 284 {
287 - if (layerDefn.GetFieldDefn(idx)->GetType() != OFTInteger)  
288 - itkExceptionMacro("Field name "<< GetParameterStringList("featout") << " already exists with a different type!");  
289 - }  
290 - else  
291 - {  
292 - OGRFieldDefn predictedField(GetParameterStringList("featout").c_str(), OFTInteger);  
293 - ogr::FieldDefn predictedFieldDef(predictedField);  
294 - outLayer.CreateField(predictedFieldDef); 285 + OGRFeatureDefn &layerDefn = outLayer.GetLayerDefn();
  286 + int idx = layerDefn.GetFieldIndex(GetParameterStringList("featout")[i].c_str());
  287 +
  288 + if (idx >= 0)
  289 + {
  290 + if (layerDefn.GetFieldDefn(idx)->GetType() != OFTReal)
  291 + itkExceptionMacro("Field name "<< GetParameterStringList("featout")[i] << " already exists with a different type!");
  292 + }
  293 + else
  294 + {
  295 + OGRFieldDefn predictedField(GetParameterStringList("featout")[i].c_str(), OFTReal);
  296 + ogr::FieldDefn predictedFieldDef(predictedField);
  297 + outLayer.CreateField(predictedFieldDef);
  298 + }
295 } 299 }
296 300
297 // Fill output layer 301 // Fill output layer
298 302
299 unsigned int count=0; 303 unsigned int count=0;
300 - std::string classfieldname = GetParameterStringList("featout"); 304 + auto classfieldname = GetParameterStringList("featout");
301 it = layer.cbegin(); 305 it = layer.cbegin();
302 itEnd = layer.cend(); 306 itEnd = layer.cend();
303 for( ; it!=itEnd ; ++it, ++count) 307 for( ; it!=itEnd ; ++it, ++count)
@@ -305,8 +309,9 @@ class CbDimensionalityReductionVector : public Application @@ -305,8 +309,9 @@ class CbDimensionalityReductionVector : public Application
305 ogr::Feature dstFeature(outLayer.GetLayerDefn()); 309 ogr::Feature dstFeature(outLayer.GetLayerDefn());
306 dstFeature.SetFrom( *it , TRUE); 310 dstFeature.SetFrom( *it , TRUE);
307 dstFeature.SetFID(it->GetFID()); 311 dstFeature.SetFID(it->GetFID());
308 - dstFeature[classfieldname].SetValue<int>(target->GetMeasurementVector(count)[0]);  
309 - 312 + for (std::size_t i=0; i<classfieldname.size(); ++i){
  313 + dstFeature[classfieldname[i]].SetValue<ValueType>(target->GetMeasurementVector(count)[i]);
  314 + }
310 if (updateMode) 315 if (updateMode)
311 { 316 {
312 outLayer.SetFeature(dstFeature); 317 outLayer.SetFeature(dstFeature);
@@ -316,6 +321,7 @@ class CbDimensionalityReductionVector : public Application @@ -316,6 +321,7 @@ class CbDimensionalityReductionVector : public Application
316 outLayer.CreateFeature(dstFeature); 321 outLayer.CreateFeature(dstFeature);
317 } 322 }
318 } 323 }
  324 +
319 if(outLayer.ogr().TestCapability("Transactions")) 325 if(outLayer.ogr().TestCapability("Transactions"))
320 { 326 {
321 const OGRErr errCommitX = outLayer.ogr().CommitTransaction(); 327 const OGRErr errCommitX = outLayer.ogr().CommitTransaction();
@@ -326,7 +332,7 @@ class CbDimensionalityReductionVector : public Application @@ -326,7 +332,7 @@ class CbDimensionalityReductionVector : public Application
326 } 332 }
327 output->SyncToDisk(); 333 output->SyncToDisk();
328 clock_t toc = clock(); 334 clock_t toc = clock();
329 - otbAppLogINFO( "Elapsed: "<< ((double)(toc - tic) / CLOCKS_PER_SEC)<<" seconds.");*/ 335 + otbAppLogINFO( "Elapsed: "<< ((double)(toc - tic) / CLOCKS_PER_SEC)<<" seconds.");
330 } 336 }
331 337
332 ModelPointerType m_Model; 338 ModelPointerType m_Model;
include/AutoencoderModel.h
@@ -34,17 +34,17 @@ public: @@ -34,17 +34,17 @@ public:
34 itkGetMacro(NumberOfIterations,unsigned int); 34 itkGetMacro(NumberOfIterations,unsigned int);
35 itkSetMacro(NumberOfIterations,unsigned int); 35 itkSetMacro(NumberOfIterations,unsigned int);
36 36
37 - itkGetMacro(Regularization,double);  
38 - itkSetMacro(Regularization,double); 37 + itkGetMacro(Regularization,itk::Array<double>);
  38 + itkSetMacro(Regularization,itk::Array<double>);
39 39
40 - itkGetMacro(Noise,double);  
41 - itkSetMacro(Noise,double); 40 + itkGetMacro(Noise,itk::Array<double>);
  41 + itkSetMacro(Noise,itk::Array<double>);
42 42
43 - itkGetMacro(rho,double);  
44 - itkSetMacro(rho,double); 43 + itkGetMacro(Rho,itk::Array<double>);
  44 + itkSetMacro(Rho,itk::Array<double>);
45 45
46 - itkGetMacro(beta,double);  
47 - itkSetMacro(beta,double); 46 + itkGetMacro(Beta,itk::Array<double>);
  47 + itkSetMacro(Beta,itk::Array<double>);
48 48
49 bool CanReadFile(const std::string & filename); 49 bool CanReadFile(const std::string & filename);
50 bool CanWriteFile(const std::string & filename); 50 bool CanWriteFile(const std::string & filename);
@@ -53,7 +53,8 @@ public: @@ -53,7 +53,8 @@ public:
53 void Load(const std::string & filename, const std::string & name="") ITK_OVERRIDE; 53 void Load(const std::string & filename, const std::string & name="") ITK_OVERRIDE;
54 54
55 void Train() ITK_OVERRIDE; 55 void Train() ITK_OVERRIDE;
56 - void TrainOneLayer(unsigned int, shark::Data<shark::RealVector> &); 56 + void TrainOneLayer(unsigned int,double, double, shark::Data<shark::RealVector> &);
  57 + void TrainOneSparseLayer(unsigned int,double, double,double, shark::Data<shark::RealVector> &);
57 58
58 protected: 59 protected:
59 AutoencoderModel(); 60 AutoencoderModel();
@@ -71,10 +72,10 @@ private: @@ -71,10 +72,10 @@ private:
71 itk::Array<unsigned int> m_NumberOfHiddenNeurons; 72 itk::Array<unsigned int> m_NumberOfHiddenNeurons;
72 /** Training parameters */ 73 /** Training parameters */
73 unsigned int m_NumberOfIterations; 74 unsigned int m_NumberOfIterations;
74 - double m_Regularization; // L2 Regularization parameter  
75 - double m_Noise; // probability for an input to be set to 0 (denosing autoencoder)  
76 - double m_rho; // Sparsity parameter  
77 - double m_beta; // Sparsity regularization parameter 75 + itk::Array<double> m_Regularization; // L2 Regularization parameter
  76 + itk::Array<double> m_Noise; // probability for an input to be set to 0 (denosing autoencoder)
  77 + itk::Array<double> m_Rho; // Sparsity parameter
  78 + itk::Array<double> m_Beta; // Sparsity regularization parameter
78 }; 79 };
79 } // end namespace otb 80 } // end namespace otb
80 81
include/AutoencoderModel.txx
@@ -20,7 +20,6 @@ @@ -20,7 +20,6 @@
20 namespace otb 20 namespace otb
21 { 21 {
22 22
23 -  
24 template <class TInputValue, class AutoencoderType> 23 template <class TInputValue, class AutoencoderType>
25 AutoencoderModel<TInputValue,AutoencoderType>::AutoencoderModel() 24 AutoencoderModel<TInputValue,AutoencoderType>::AutoencoderModel()
26 { 25 {
@@ -42,31 +41,34 @@ void AutoencoderModel&lt;TInputValue,AutoencoderType&gt;::Train() @@ -42,31 +41,34 @@ void AutoencoderModel&lt;TInputValue,AutoencoderType&gt;::Train()
42 41
43 for (unsigned int i = 0 ; i < m_NumberOfHiddenNeurons.Size(); ++i) 42 for (unsigned int i = 0 ; i < m_NumberOfHiddenNeurons.Size(); ++i)
44 { 43 {
45 - TrainOneLayer( m_NumberOfHiddenNeurons[i], inputSamples); 44 + if (m_Noise[i] != 0) // Shark doesn't allow to train a layer using a sparsity term AND a noisy input. (shark::SparseAutoencoderError takes an autoen
  45 + {
  46 + TrainOneLayer( m_NumberOfHiddenNeurons[i],m_Noise[i],m_Regularization[i], inputSamples);
  47 + }
  48 + else
  49 + {
  50 + TrainOneSparseLayer( m_NumberOfHiddenNeurons[i],m_Rho[i],m_Beta[i],m_Regularization[i], inputSamples);
  51 + }
46 } 52 }
47 } 53 }
48 54
49 template <class TInputValue, class AutoencoderType> 55 template <class TInputValue, class AutoencoderType>
50 -void AutoencoderModel<TInputValue,AutoencoderType>::TrainOneLayer(unsigned int nbneuron, shark::Data<shark::RealVector> &samples) 56 +void AutoencoderModel<TInputValue,AutoencoderType>::TrainOneLayer(unsigned int nbneuron,double noise_strength,double regularization, shark::Data<shark::RealVector> &samples)
51 { 57 {
52 AutoencoderType net; 58 AutoencoderType net;
53 - /*std::vector<shark::RealVector> features;  
54 -  
55 - Shark::ListSampleToSharkVector(this->GetInputListSample(), features);  
56 -  
57 - shark::Data<shark::RealVector> inputSamples = shark::createDataFromRange( features );  
58 - */ //in Train() now 59 +
59 std::size_t inputs = dataDimension(samples); 60 std::size_t inputs = dataDimension(samples);
60 net.setStructure(inputs, nbneuron); 61 net.setStructure(inputs, nbneuron);
61 initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs)); 62 initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
62 - shark::ImpulseNoiseModel noise(m_Noise,0.0); //set an input pixel with probability m_Noise to 0 63 + shark::ImpulseNoiseModel noise(noise_strength,0.0); //set an input pixel with probability m_Noise to 0
63 shark::ConcatenatedModel<shark::RealVector,shark::RealVector> model = noise>> net; 64 shark::ConcatenatedModel<shark::RealVector,shark::RealVector> model = noise>> net;
64 shark::LabeledData<shark::RealVector,shark::RealVector> trainSet(samples,samples);//labels identical to inputs 65 shark::LabeledData<shark::RealVector,shark::RealVector> trainSet(samples,samples);//labels identical to inputs
65 shark::SquaredLoss<shark::RealVector> loss; 66 shark::SquaredLoss<shark::RealVector> loss;
66 shark::ErrorFunction error(trainSet, &model, &loss); 67 shark::ErrorFunction error(trainSet, &model, &loss);
67 - //shark::SparseAutoencoderError error(data,&model, &loss, m_rho, m_beta); 68 + //shark::SparseAutoencoderError error(trainSet,&model, &loss, m_Rho, m_Beta);
  69 + //shark::SparseAutoencoderError error(trainSet,&net, &loss, 0.1, 0.1);
68 shark::TwoNormRegularizer regularizer(error.numberOfVariables()); 70 shark::TwoNormRegularizer regularizer(error.numberOfVariables());
69 - error.setRegularizer(m_Regularization,&regularizer); 71 + error.setRegularizer(regularization,&regularizer);
70 72
71 shark::IRpropPlusFull optimizer; 73 shark::IRpropPlusFull optimizer;
72 error.init(); 74 error.init();
@@ -83,6 +85,35 @@ void AutoencoderModel&lt;TInputValue,AutoencoderType&gt;::TrainOneLayer(unsigned int n @@ -83,6 +85,35 @@ void AutoencoderModel&lt;TInputValue,AutoencoderType&gt;::TrainOneLayer(unsigned int n
83 85
84 86
85 template <class TInputValue, class AutoencoderType> 87 template <class TInputValue, class AutoencoderType>
  88 +void AutoencoderModel<TInputValue,AutoencoderType>::TrainOneSparseLayer(unsigned int nbneuron,double rho,double beta, double regularization, shark::Data<shark::RealVector> &samples)
  89 +{
  90 + AutoencoderType net;
  91 +
  92 + std::size_t inputs = dataDimension(samples);
  93 + net.setStructure(inputs, nbneuron);
  94 + initRandomUniform(net,-0.1*std::sqrt(1.0/inputs),0.1*std::sqrt(1.0/inputs));
  95 +
  96 + shark::LabeledData<shark::RealVector,shark::RealVector> trainSet(samples,samples);//labels identical to inputs
  97 + shark::SquaredLoss<shark::RealVector> loss;
  98 + shark::SparseAutoencoderError error(trainSet,&net, &loss, rho, beta);
  99 +
  100 + shark::TwoNormRegularizer regularizer(error.numberOfVariables());
  101 + error.setRegularizer(regularization,&regularizer);
  102 +
  103 + shark::IRpropPlusFull optimizer;
  104 + error.init();
  105 + optimizer.init(error);
  106 + std::cout<<"Optimizing model: "+net.name()<<std::endl;
  107 + for(std::size_t i = 0; i != m_NumberOfIterations; ++i){
  108 + optimizer.step(error);
  109 + std::cout<<i<<" "<<optimizer.solution().value<<std::endl;
  110 + }
  111 + net.setParameterVector(optimizer.solution().point);
  112 + m_net.push_back(net);
  113 + samples = net.encode(samples);
  114 +}
  115 +
  116 +template <class TInputValue, class AutoencoderType>
86 bool AutoencoderModel<TInputValue,AutoencoderType>::CanReadFile(const std::string & filename) 117 bool AutoencoderModel<TInputValue,AutoencoderType>::CanReadFile(const std::string & filename)
87 { 118 {
88 try 119 try
include/cbTrainAutoencoder.txx
@@ -55,16 +55,24 @@ cbLearningApplicationBaseDR&lt;TInputValue,TOutputValue&gt; @@ -55,16 +55,24 @@ cbLearningApplicationBaseDR&lt;TInputValue,TOutputValue&gt;
55 "The number of neurons in each hidden layer."); 55 "The number of neurons in each hidden layer.");
56 56
57 //Regularization 57 //Regularization
58 - AddParameter(ParameterType_Float, "model.autoencoder.regularization", "Strength of the regularization");  
59 - SetParameterFloat("model.autoencoder.regularization",0, false); 58 + AddParameter(ParameterType_StringList, "model.autoencoder.regularization", "Strength of the regularization");
60 SetParameterDescription("model.autoencoder.regularization", 59 SetParameterDescription("model.autoencoder.regularization",
61 "Strength of the L2 regularization used during training"); 60 "Strength of the L2 regularization used during training");
62 61
63 //Noise strength 62 //Noise strength
64 - AddParameter(ParameterType_Float, "model.autoencoder.noise", "Strength of the noise");  
65 - SetParameterFloat("model.autoencoder.noise",0, false); 63 + AddParameter(ParameterType_StringList, "model.autoencoder.noise", "Strength of the noise");
66 SetParameterDescription("model.autoencoder.noise", 64 SetParameterDescription("model.autoencoder.noise",
67 "Strength of the noise"); 65 "Strength of the noise");
  66 +
  67 + // Sparsity parameter
  68 + AddParameter(ParameterType_StringList, "model.autoencoder.rho", "Sparsity parameter");
  69 + SetParameterDescription("model.autoencoder.rho",
  70 + "Sparsity parameter");
  71 +
  72 + // Sparsity regularization strength
  73 + AddParameter(ParameterType_StringList, "model.autoencoder.beta", "Sparsity regularization strength");
  74 + SetParameterDescription("model.autoencoder.beta",
  75 + "Sparsity regularization strength");
68 } 76 }
69 77
70 78
@@ -102,16 +110,34 @@ void cbLearningApplicationBaseDR&lt;TInputValue,TOutputValue&gt; @@ -102,16 +110,34 @@ void cbLearningApplicationBaseDR&lt;TInputValue,TOutputValue&gt;
102 { 110 {
103 typename autoencoderchoice::Pointer dimredTrainer = autoencoderchoice::New(); 111 typename autoencoderchoice::Pointer dimredTrainer = autoencoderchoice::New();
104 itk::Array<unsigned int> nb_neuron; 112 itk::Array<unsigned int> nb_neuron;
105 - std::vector<std::basic_string<char>> s= GetParameterStringList("model.autoencoder.nbneuron");  
106 - nb_neuron.SetSize(s.size());  
107 - for (int i=0; i<s.size(); i++){ // This will be templated later (the 3)  
108 - nb_neuron[i]=std::stoi(s[i]); 113 + itk::Array<float> noise;
  114 + itk::Array<float> regularization;
  115 + itk::Array<float> rho;
  116 + itk::Array<float> beta;
  117 + std::vector<std::basic_string<char>> s_nbneuron= GetParameterStringList("model.autoencoder.nbneuron");
  118 + std::vector<std::basic_string<char>> s_noise= GetParameterStringList("model.autoencoder.noise");
  119 + std::vector<std::basic_string<char>> s_regularization= GetParameterStringList("model.autoencoder.regularization");
  120 + std::vector<std::basic_string<char>> s_rho= GetParameterStringList("model.autoencoder.rho");
  121 + std::vector<std::basic_string<char>> s_beta= GetParameterStringList("model.autoencoder.beta");
  122 + nb_neuron.SetSize(s_nbneuron.size());
  123 + noise.SetSize(s_nbneuron.size());
  124 + regularization.SetSize(s_nbneuron.size());
  125 + rho.SetSize(s_nbneuron.size());
  126 + beta.SetSize(s_nbneuron.size());
  127 + for (int i=0; i<s_nbneuron.size(); i++){
  128 + nb_neuron[i]=std::stoi(s_nbneuron[i]);
  129 + noise[i]=std::stof(s_noise[i]);
  130 + regularization[i]=std::stof(s_regularization[i]);
  131 + rho[i]=std::stof(s_rho[i]);
  132 + beta[i]=std::stof(s_beta[i]);
109 } 133 }
110 std::cout << nb_neuron << std::endl; 134 std::cout << nb_neuron << std::endl;
111 dimredTrainer->SetNumberOfHiddenNeurons(nb_neuron); 135 dimredTrainer->SetNumberOfHiddenNeurons(nb_neuron);
112 dimredTrainer->SetNumberOfIterations(GetParameterInt("model.autoencoder.nbiter")); 136 dimredTrainer->SetNumberOfIterations(GetParameterInt("model.autoencoder.nbiter"));
113 - dimredTrainer->SetRegularization(GetParameterFloat("model.autoencoder.regularization"));  
114 - dimredTrainer->SetNoise(GetParameterFloat("model.autoencoder.noise")); 137 + dimredTrainer->SetRegularization(regularization);
  138 + dimredTrainer->SetNoise(noise);
  139 + dimredTrainer->SetRho(rho);
  140 + dimredTrainer->SetBeta(beta);
115 dimredTrainer->SetInputListSample(trainingListSample); 141 dimredTrainer->SetInputListSample(trainingListSample);
116 std::cout << "before train" << std::endl; 142 std::cout << "before train" << std::endl;
117 dimredTrainer->Train(); 143 dimredTrainer->Train();