Commit b9e0e21e66d80626debfc6ec23fb93ce21a6cdcc

Authored by Louis Baetens
1 parent cde08783
Exists in master

REFAC PCC, more comments and main() cleaner

PCC/alcd_labellisation_posttreatment.py
... ... @@ -8,9 +8,10 @@ import otbApplication
8 8  
9 9  
10 10 def binary_label_dilatation(binary_label_tif, out_tif, radius, regularization = True):
11   - ''' Create a dilated contour of the given class
  11 + ''' Create a dilation
12 12 Class 1 is background, 2 is clouds
13 13 '''
  14 + # Can regularize the map beforehand to avoid single pixels being dilated
14 15 if regularization:
15 16 ClassificationMapRegularization = otbApplication.Registry.CreateApplication("ClassificationMapRegularization")
16 17 ClassificationMapRegularization.SetParameterString("io.in", str(binary_label_tif))
... ... @@ -45,7 +46,7 @@ def binary_label_dilatation(binary_label_tif, out_tif, radius, regularization =
45 46  
46 47  
47 48 def binary_label_erosion(binary_label_tif, out_tif, radius):
48   - ''' Create a dilated contour of the given class
  49 + ''' Create an erosion
49 50 Class 1 is background, 2 is clouds
50 51 '''
51 52  
... ... @@ -72,14 +73,11 @@ def thresholded_clouds_conversion(raw_img, multi_label_tif, out_tif, threshold):
72 73 Change the classification of the 'high clouds' into 'land' if the
73 74 1375nm band is less than the threshold in the raw_img
74 75 The higher the threshold, the more cirrus will be filtered out
75   -
76   - PROBLEM : DO NOT OCCUPY THE SAME INPUT SPACE, PLEASE DO A SUPERIMPOSE BEFORE
77 76 '''
78   -
  77 + # superimpose to realign the tif
79 78 Superimpose = otbApplication.Registry.CreateApplication("Superimpose")
80 79 Superimpose.SetParameterString("inr", str(multi_label_tif))
81 80 Superimpose.SetParameterString("inm", str(raw_img))
82   - #~ Superimpose.SetParameterString("out", str(out_tif))
83 81 Superimpose.SetParameterString("interpolator", "nn")
84 82  
85 83 Superimpose.UpdateParameters()
... ... @@ -88,11 +86,8 @@ def thresholded_clouds_conversion(raw_img, multi_label_tif, out_tif, threshold):
88 86  
89 87  
90 88 BandMathX = otbApplication.Registry.CreateApplication("BandMathX")
91   -
92 89 BandMathX.AddImageToParameterInputImageList("il", Superimpose.GetParameterOutputImage("out"))
93 90 BandMathX.AddParameterStringList("il", str(multi_label_tif))
94   -
95   -
96 91 BandMathX.SetParameterString("out", str(out_tif))
97 92  
98 93 # Band 10 is the 1375nm band, class 3 is high clouds, 5 is land
... ... @@ -106,24 +101,9 @@ def thresholded_clouds_conversion(raw_img, multi_label_tif, out_tif, threshold):
106 101  
107 102  
108 103  
109   -def main():
110   -
111   - #~ in_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/Mongu_34LGJ_20171013/Binary_classif/binary_fmask_Mongu_20171013.tif'
112   - #~ out_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/Mongu_34LGJ_20171013/Binary_classif/binary_fmask_DILATED_Mongu_20171013.tif'
113   - #~ radius = 12*3
114   - #~ binary_label_dilatation(in_tif, out_tif, radius, regularization = False)
115   -
116   - #~ return
117   -
  104 +def main():
118 105 comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
119 106  
120   - raw_img = '/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Arles_31TFJ_20171002/In_data/Image/Arles_bands.tif'
121   - multi_label_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_radius4/Arles_31TFJ_20171002/Multi_classif/multi_alcd_initial_Arles_20171002.tif'
122   - out_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_radius4/Arles_31TFJ_20171002/Multi_classif/thresh_multi_alcd_Arles_20171002.tif'
123   - thresholded_clouds_conversion(raw_img, multi_label_tif, out_tif, threshold = 100.)
124   -
125   - return
126   -
127 107 binary_label_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/Orleans_20170516/Binary_classif/binary_alcd_Orleans_20170516.tif'
128 108 out_tif = '/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/Orleans_20170516/Binary_classif/dilat_binary_alcd_Orleans_20170516.tif'
129 109 binary_label_dilatation(binary_label_tif, out_tif, radius = 4)
... ...
PCC/all_run_pcc.py
... ... @@ -10,7 +10,6 @@ import glob
10 10 import argparse
11 11 import time
12 12  
13   -
14 13 import masks_conversion
15 14 import find_chain_directory_paths
16 15 import comparison
... ... @@ -31,8 +30,8 @@ def create_directories(comparison_parameters):
31 30 os.makedirs(to_check)
32 31 print(to_check + ' created')
33 32  
  33 +
34 34 main_dir = comparison_parameters["user_choices"]["main_dir"]
35   -
36 35 directories = ['', 'Multi_classif', 'Binary_classif',
37 36 'Multi_difference', 'Binary_difference', 'Original_data',
38 37 'Intermediate', 'Statistics', 'Out', op.join('Statistics', 'features')]
... ... @@ -126,8 +125,8 @@ def run_all(part, location=None, current_date=None, masks_already_computed = Fal
126 125 # sen2cor
127 126 sen2cor_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='sen2cor', mask_type='cloud', display = False)
128 127 shutil.copy(sen2cor_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["sen2cor"]["cloud_mask"]))
129   - except Exception as exception:
130   - print(exception)
  128 + except:
  129 + logging.exception('')
131 130  
132 131 try:
133 132 # maja
... ... @@ -135,8 +134,8 @@ def run_all(part, location=None, current_date=None, masks_already_computed = Fal
135 134 shutil.copy(maja_cloud_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["maja"]["cloud_mask"]))
136 135 maja_geo_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='maja', mask_type='geo', display = False)
137 136 shutil.copy(maja_geo_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["maja"]["geo_mask"]))
138   - except Exception as exception:
139   - print(exception)
  137 + except:
  138 + logging.exception('')
140 139  
141 140 try:
142 141 # fmask
... ... @@ -145,16 +144,17 @@ def run_all(part, location=None, current_date=None, masks_already_computed = Fal
145 144 else:
146 145 fmask_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='fmask', mask_type='cloud', display = False)
147 146 shutil.copy(fmask_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["fmask"]["cloud_mask"]))
148   - except Exception as exception:
149   - print(exception)
  147 + except:
  148 + logging.exception('')
  149 +
150 150  
151 151 # original labeled ALCD
152 152 try:
153 153 alcd_classif_name = comparison_parameters["alcd_output"]["labeled_img_name"]
154 154 alcd_output = op.join(Data_ALCD_dir, (location + '_' + tile + '_' + date_string), 'Out', alcd_classif_name)
155 155 shutil.copy(alcd_output, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["alcd_initial"]["cloud_mask"]))
156   - except Exception as exception:
157   - print(exception)
  156 + except:
  157 + logging.exception('')
158 158  
159 159  
160 160 elif part == 3:
... ... @@ -184,16 +184,14 @@ def run_all(part, location=None, current_date=None, masks_already_computed = Fal
184 184 metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
185 185 except:
186 186 logging.exception('')
187   - #~ for processing_chain in ['maja', 'sen2cor', 'fmask']:
188   - #~ pixels_features_analysis.plot_confusion_repartition(comparison_parameters, processing_chain)
  187 +
189 188 if 'd' in alcd_ref:
190 189 algo = 'alcd_dilat'
191 190 try:
192 191 metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
193 192 except:
194 193 logging.exception('')
195   - #~ for processing_chain in ['maja', 'sen2cor', 'fmask']:
196   - #~ pixels_features_analysis.plot_confusion_repartition(comparison_parameters, processing_chain)
  194 +
197 195  
198 196 elif part == 6:
199 197 # ----- Post treat the difference masks
... ... @@ -234,7 +232,6 @@ def main():
234 232 if alcd_ref == None:
235 233 alcd_ref = 'id'
236 234  
237   -
238 235 if results.binary_only == None:
239 236 binary_only = False
240 237 else :
... ...
PCC/comparison.py
... ... @@ -15,7 +15,7 @@ logging.basicConfig(filename='log/pcc_run.log',level=logging.ERROR, format='%(as
15 15 def compute_comparative_image(reference_tif, to_compare_tif, out_tif, binary = True):
16 16 '''
17 17 Compute the difference between a reference and another image
18   - It can be assimiliated to a confusion matrix
  18 + It can be assimiliated to the vizualisation of a confusion matrix
19 19 '''
20 20 BandMathX = otbApplication.Registry.CreateApplication("BandMathX")
21 21  
... ... @@ -30,7 +30,7 @@ def compute_comparative_image(reference_tif, to_compare_tif, out_tif, binary = T
30 30 expressions.append("( (im1b1 == 2 && im2b1 == 1) ? 2 : 0)") # false negative
31 31 expressions.append("( (im1b1 == 1 && im2b1 == 2) ? 3 : 0)") # false positive
32 32 expressions.append("( (im1b1 == 1 && im2b1 == 1) ? 4 : 0)") # true negative
33   - # 1 : background, 2 : clouds
  33 + # Or in 2 classes (1 : background, 2 : clouds)
34 34 else:
35 35 expressions.append("( (im1b1 == im2b1 and im1b1 != 0) ? 1 : 0)") # true positive
36 36 expressions.append("( (im1b1 != im2b1 and im1b1 != 0) ? 2 : 0)") # false negative
... ... @@ -51,13 +51,13 @@ def compute_comparative_image(reference_tif, to_compare_tif, out_tif, binary = T
51 51  
52 52 def compute_confusion_matrix(difference_tif, binary = True, save_file = ''):
53 53 '''
54   - Compute the confusion matrix for all the pixels between
55   - an image and a reference one
56   - They need to have the same resolution
  54 + Compute the confusion matrix from a difference_tif computed with
  55 + compute_comparative_image
57 56 '''
58   - # Load the images as arrays
  57 + # Load the image as array
59 58 diff_array = np.asarray(Image.open(difference_tif))
60 59  
  60 + # Stretch it to a vector
61 61 ligs, cols = diff_array.shape
62 62 diff_array = np.reshape(diff_array, (ligs*cols))
63 63  
... ... @@ -193,7 +193,7 @@ def compare_all(comparison_parameters, binary_classif = True, reference_algo = '
193 193 confusion_matrix_file = op.join(main_dir, 'Statistics', reference_sub_dir, ('stats_' + op.basename(to_compare_tifs[k])[0:-4] + '.json'))
194 194 compute_confusion_matrix(difference_tifs[k], binary = binary_classif, save_file = confusion_matrix_file)
195 195 except:
196   - pass
  196 + logging.exception('')
197 197  
198 198  
199 199 def main():
... ...
PCC/find_chain_directory_paths.py
1 1 #!/usr/bin/python
2 2 # -*- coding: utf-8 -*-
3 3  
4   -
5 4 import os.path as op
6 5 import glob
7 6 import json
... ... @@ -9,7 +8,7 @@ import json
9 8  
10 9 def get_all_dates(location, processing_chain='maja', display = True):
11 10 '''
12   - Get all available dates for a given location
  11 + Get all available dates for a given location and processing chain
13 12 '''
14 13 # Create all the possible dates
15 14 years = range(2015,2020)
... ... @@ -30,7 +29,6 @@ def get_all_dates(location, processing_chain='maja', display = True):
30 29 except:
31 30 pass
32 31  
33   -
34 32 valid_dates = sorted(valid_dates)
35 33 if display:
36 34 print(valid_dates)
... ... @@ -42,13 +40,10 @@ def check_existing_date(location, date_to_verify, processing_chain='maja', displ
42 40 Check if a date is valid for a chain
43 41 '''
44 42 all_valid_dates = get_all_dates(location, processing_chain, display = False)
45   - if date_to_verify in all_valid_dates:
46   - date_is_valid = True
47   - else:
48   - date_is_valid = False
49   -
  43 + date_is_valid = date_to_verify in all_valid_dates
50 44 if display:
51 45 print(date_is_valid)
  46 +
52 47 return date_is_valid
53 48  
54 49  
... ... @@ -112,7 +107,7 @@ def get_processing_dir(location, date_string, processing_chain='maja', display =
112 107 def get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = True):
113 108 '''
114 109 Return the full path of the masks returned by the chain
115   - mask_type is for maja, which can be the cloud mask or the geo mask
  110 + mask_type is for MAJA, which can be the cloud mask or the geo mask
116 111 '''
117 112  
118 113 if processing_chain == 'maja':
... ... @@ -161,10 +156,9 @@ def get_mask_path(location, date_string, processing_chain='maja', mask_type='clo
161 156  
162 157 def main():
163 158 location = 'Gobabeb'
164   - date_string = '20180209'
165   - check_existing_date(location, date_string, processing_chain='sen2cor', display = True)
166   -
167   - #~ get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = True)
  159 + date_string = '20170909'
  160 + check_existing_date(location, date_string, processing_chain='maja', display = True)
  161 + get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = True)
168 162 return
169 163  
170 164  
... ...
PCC/metrics_grapher.py
... ... @@ -50,7 +50,7 @@ def plot_all_metrics(comparison_parameters, reference_algo = 'alcd_initial'):
50 50  
51 51  
52 52 # data to plot
53   - # should have 3 precessing chains
  53 + # should have 3 processing chains
54 54 n_processing_chains = len(accuracies)
55 55  
56 56 # create plot
... ...
PCC/pixels_features_analysis.py
... ... @@ -14,7 +14,7 @@ def extract_class_band(raw_img, band_to_extract, diff_img, diff_class_nb, alcd_l
14 14 From the difference image (e.g. diff_binary) between the processing
15 15 chain and the ALCD, and the ALCD labeled, extract the data for the pixels
16 16 in the class_nb for both images
17   - Mostly used to plot histograms for each class and band
  17 + Mostly used to plot histograms for each class and band
18 18 '''
19 19  
20 20 BandMathX = otbApplication.Registry.CreateApplication("BandMathX")
... ... @@ -44,7 +44,10 @@ def extract_class_band(raw_img, band_to_extract, diff_img, diff_class_nb, alcd_l
44 44  
45 45 def count_misclassified_class(diff_img, diff_class_nb, alcd_labeled_img, alcd_class_nb):
46 46 '''
47   -
  47 + Count the number of pixels being at the same time of the
  48 + diff_class_nb and alcd_class_nb.
  49 + e.g. if diff_class_nb = 1 and alcd_class_nb = 2, it will returns the
  50 + number of true positive and low_clouds pixels.
48 51 '''
49 52  
50 53 Superimpose = otbApplication.Registry.CreateApplication("Superimpose")
... ... @@ -54,13 +57,7 @@ def count_misclassified_class(diff_img, diff_class_nb, alcd_labeled_img, alcd_cl
54 57 Superimpose.UpdateParameters()
55 58 Superimpose.Execute()
56 59  
57   -
58   -
59 60  
60   -
61   -
62   -
63   -
64 61 BandMathX = otbApplication.Registry.CreateApplication("BandMathX")
65 62  
66 63 BandMathX.AddParameterStringList("il", str(diff_img))
... ... @@ -83,6 +80,11 @@ def count_misclassified_class(diff_img, diff_class_nb, alcd_labeled_img, alcd_cl
83 80  
84 81  
85 82 def plot_confusion_repartition(comparison_parameters, processing_chain):
  83 + '''
  84 + Plot the repartition of classes for each cell of the confusion
  85 + matrix.
  86 + e.g. what percentage of low_clouds and clouds_shadows in false positive
  87 + '''
86 88 main_dir = comparison_parameters["user_choices"]["main_dir"]
87 89 location = comparison_parameters["user_choices"]["location"]
88 90 date = comparison_parameters["user_choices"]["current_date"]
... ... @@ -156,8 +158,8 @@ def plot_confusion_repartition(comparison_parameters, processing_chain):
156 158 plt.tight_layout()
157 159  
158 160 # save the figure
159   - #~ plt.show(block=False)
160   - out_fig = op.join(main_dir, 'Statistics', 'repartition_matconf_{}_{}_{}.png'.format(processing_chain, location, date))
  161 + #~ plt.show()
  162 + out_fig = op.join(main_dir, 'Statistics', 'features', 'repartition_matconf_{}_{}_{}.png'.format(processing_chain, location, date))
161 163 plt.savefig(out_fig, bbox_inches='tight')
162 164  
163 165  
... ... @@ -258,7 +260,7 @@ def get_miclassified_histograms(comparison_parameters, processing_chain, diff_cl
258 260 #~ plt.legend()
259 261  
260 262 plt.tight_layout()
261   - #~ plt.show(block=False)
  263 + plt.show()
262 264  
263 265 dict_diff_class_nb_short={1:'TP', 2:'FN', 3:'FP', 4:'TN'}
264 266 out_fig = op.join(main_dir, 'Statistics',
... ... @@ -292,7 +294,7 @@ def get_normalized_histo(comparison_parameters, processing_chain, diff_class_nb,
292 294 raw_img = glob.glob(op.join(original_data, location_dir, 'In_data', 'Image', (location + '_bands.tif')))[0]
293 295 main_dir = comparison_parameters["user_choices"]["main_dir"]
294 296  
295   - diff_img = glob.glob(op.join(main_dir, 'Binary_difference', '*{}*.tif'.format(processing_chain)))[0]
  297 + diff_img = glob.glob(op.join(main_dir, 'Binary_difference', 'ALCD_initial', '*{}*.tif'.format(processing_chain)))[0]
296 298 alcd_labeled_img = op.join(main_dir, 'Original_data', comparison_parameters["processing"]["alcd_initial"]["cloud_mask"])
297 299  
298 300  
... ... @@ -347,7 +349,7 @@ def extract_bands_per_class(comparison_parameters, processing_chain, alcd_class_
347 349 '''
348 350 Extract the bands for all the classes and plot an histogram of every one
349 351  
350   - TODO : change 'bands_nb' and 'bands' by 'features' and 'features_nb'
  352 + DEPRECATED, should be improved !
351 353 '''
352 354 main_dir = comparison_parameters["user_choices"]["main_dir"]
353 355 stats_dir = op.join(main_dir, 'Statistics')
... ... @@ -391,9 +393,9 @@ def extract_bands_per_class(comparison_parameters, processing_chain, alcd_class_
391 393  
392 394 plt.title('True class: {}\nFeature {}'.format(alcd_class_dict[str(alcd_class_nb)], feat))
393 395 plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
394   - #~ plt.show(block = False)
  396 + plt.show()
395 397 out_fig = op.join(main_dir, 'Statistics', 'features', 'class_{}_feature_{}.png'.format(alcd_class_nb, feat))
396   - plt.savefig(out_fig, bbox_inches='tight')
  398 + #~ plt.savefig(out_fig, bbox_inches='tight')
397 399  
398 400 #~ plt.show(block = False)
399 401  
... ... @@ -448,44 +450,20 @@ def create_colormap(color_pattern = 'diff', class_call = 'number'):
448 450  
449 451  
450 452 def main():
451   - comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
452   - processing_chain = 'sen2cor'
  453 + comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
453 454  
454 455 for processing_chain in ['maja', 'sen2cor', 'fmask']:
455   - plot_confusion_repartition(comparison_parameters, processing_chain)
456   -
457   - return
458   -
459   -
460   - diff_class_nb = 2
461   - alcd_class_nb = 2
462   -
463   - plot_confusion_repartition(comparison_parameters, processing_chain)
464   - #~ extract_bands_per_class(comparison_parameters, processing_chain, alcd_class_nb)
  456 + try:
  457 + plot_confusion_repartition(comparison_parameters, processing_chain)
  458 + except:
  459 + pass
465 460  
466   - #~ a,b,c =get_miclassified_histograms(comparison_parameters, processing_chain, diff_class_nb)
467   - #~ print(a)
468   - #~ print(b)
469   - #~ print(c)
470 461 return
471   -
472   - #~ extract_bands_per_class(comparison_parameters)
473   - processing_chain = 'fmask'
474   - diff_class_nb = 2
475 462 alcd_class_nb = 2
476   - band = 2
477   - #~ get_normalized_histo(comparison_parameters, processing_chain, diff_class_nb, alcd_class_nb, band)
  463 + extract_bands_per_class(comparison_parameters, processing_chain, alcd_class_nb)
478 464  
479   - for alcd_class_nb in [1,2,3,4,5,6,7]:
480   - plt.close('all')
481   - extract_bands_per_class(comparison_parameters, processing_chain, alcd_class_nb)
482 465  
483   - return
484 466  
485   - for class_nb in np.append(np.arange(1,8), -1):
486   - features = np.arange(1,3)
487   - compute_correlation(comparison_parameters, class_nb, features)
488   - #~ plt.show()
489 467  
490 468 if __name__ == '__main__':
491 469 main()
... ...
PCC/png_converter.py
... ... @@ -15,10 +15,7 @@ def get_quicklook(location, date, out_png):
15 15 Create the quicklook of the given image
16 16 '''
17 17 print(' Creating the quicklook')
18   -
19 18 paths_configuration = json.load(open(op.join('..', 'paths_configuration.json')))
20   -
21   - #~ L1C_dir = '/mnt/data/SENTINEL2/L1C_PDGS'
22 19 L1C_dir = paths_configuration["global_chains_paths"]["L1C"]
23 20 location_dir = op.join(L1C_dir, location)
24 21  
... ... @@ -27,13 +24,9 @@ def get_quicklook(location, date, out_png):
27 24 final = glob.glob(op.join(granule, '*', 'IMG_DATA'))[0]
28 25  
29 26 bands_path = glob.glob(op.join(final, '*_B*.jp2'))
30   - for band in bands_path:
31   - if '04.jp2' in band:
32   - R = band
33   - elif '03.jp2' in band:
34   - G = band
35   - elif '02.jp2' in band:
36   - B = band
  27 + R = glob.glob(op.join(final, '*_B04.jp2'))[0]
  28 + G = glob.glob(op.join(final, '*_B03.jp2'))[0]
  29 + B = glob.glob(op.join(final, '*_B02.jp2'))[0]
37 30  
38 31 bands_text = ''
39 32 for band in [R,G,B]:
... ... @@ -78,6 +71,10 @@ def create_png(in_tifs, out_pngs, color_scheme='difference'):
78 71 return
79 72  
80 73 def add_chain_text(in_pngs, out_pngs, comparison_parameters, additional_text=''):
  74 + '''
  75 + Add a text with the chain name and additional text (e.g. the date)
  76 + to the png
  77 + '''
81 78 chain = ''
82 79 for k in range(len(in_pngs)):
83 80 in_png = in_pngs[k]
... ... @@ -104,7 +101,6 @@ def add_chain_text(in_pngs, out_pngs, comparison_parameters, additional_text='')
104 101 else:
105 102 chain = 'Unknown'
106 103  
107   -
108 104 text = chain + additional_text
109 105 add_text(in_png, out_png, comparison_parameters, text=text)
110 106  
... ... @@ -136,7 +132,6 @@ def post_treatment(comparison_parameters):
136 132 date = comparison_parameters["user_choices"]["current_date"]
137 133  
138 134 # Generate the quicklook
139   - #~ out_png = op.join(main_dir, 'Out', 'quicklook_' + location + '_' + date + '.png')
140 135 out_png = op.join(main_dir, 'Out', 'quicklook_' + location + '_' + date + '.png')
141 136 get_quicklook(location, date, out_png)
142 137  
... ... @@ -145,6 +140,7 @@ def post_treatment(comparison_parameters):
145 140 references_dir = [comparison_parameters["processing"]["alcd_initial"]["sub_dir"],
146 141 comparison_parameters["processing"]["alcd_dilat"]["sub_dir"]]
147 142  
  143 + # Generate all the differences png
148 144 for ref_dir in references_dir:
149 145 sub_bin_dir = op.join(bin_dir, ref_dir)
150 146 names = os.listdir(sub_bin_dir)
... ... @@ -165,19 +161,9 @@ def main():
165 161 return
166 162 location = 'Arles'
167 163 date = '20171002'
168   - out_png = 'out_quick.png'
  164 + out_png = 'tmp/out_quick.png'
169 165 get_quicklook(location, date, out_png)
170   -
171   - return
172   - bin_dir = '/mnt/data/home/baetensl/classification_clouds/Data_to_compare/Arles_20171002/Binary_difference'
173   - names = os.listdir(bin_dir)
174   - names = [n for n in names if (n[-4:] == '.tif')]
175   - in_tifs = [op.join(bin_dir, n) for n in names]
176   - #~ out_pngs = [op.join(bin_dir, (n[0:-4] + '.png')) for n in names]
177   - out_pngs = [op.join(bin_dir, (n[0:-4] + '.png')) for n in names]
178   - create_png(in_tifs, out_pngs, 'difference')
179   -
180   - add_chain_text(out_pngs, out_pngs)
  166 +
181 167  
182 168 if __name__ == '__main__':
183 169 main()
... ...
PCC/statistics_synthesis.py
... ... @@ -12,6 +12,9 @@ import datetime
12 12  
13 13  
14 14 def get_main_directories(paths_configuration, locations):
  15 + '''
  16 + Return the main directory of all the prescribed locations
  17 + '''
15 18 Data_PCC_dir = paths_configuration["data_paths"]["data_pcc"]
16 19 main_dirs = []
17 20 for location in locations:
... ... @@ -22,7 +25,7 @@ def get_main_directories(paths_configuration, locations):
22 25  
23 26 def get_all_stats_files(paths_configuration, locations, alcd_algo_dir_name):
24 27 '''
25   - TODO CHANGER ALCD_ALGO VERS DES MINUSCULES POUR ALLER AVEC LES NOMS DE DOSSIERS
  28 + Return the binary difference stats files from the locations
26 29 '''
27 30 main_dirs = get_main_directories(paths_configuration, locations)
28 31 potential_files = []
... ... @@ -45,6 +48,10 @@ def get_metrics_from_stats(stat_file):
45 48  
46 49  
47 50 def get_data_and_means(stats_files):
  51 + '''
  52 + Return all the data, the means of each and the standard deviation
  53 + of each from multiple stats_files
  54 + '''
48 55 metrics = []
49 56 for stat_file in stats_files:
50 57 metrics.append(get_metrics_from_stats(stat_file))
... ... @@ -56,9 +63,6 @@ def get_data_and_means(stats_files):
56 63  
57 64  
58 65  
59   -
60   -
61   -
62 66 def make_table_metrics(paths_configuration, comparison_parameters, locations, alcd_algo = 'alcd_initial',
63 67 excluded = [], maja_erode = False, cas_alcd = None,
64 68 print_all_stats = True, file_suffix = ''):
... ... @@ -66,11 +70,11 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
66 70 Make tables with the results of the chains on each scene
67 71 '''
68 72  
69   -
70 73 csv_path = '/mnt/data/home/baetensl/clouds_detection_git/Various_data/all_28_sites_sorted.csv'
71 74  
72 75 all_scenes_names = []
73 76  
  77 + # append all the potential informations
74 78 with open(csv_path, mode='r') as infile:
75 79 reader = csv.reader(infile)
76 80 reader.next()
... ... @@ -87,10 +91,10 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
87 91 scene_id += 1
88 92  
89 93  
90   -
  94 + # current ALCD reference sub dir (e.g. initial or dilated)
91 95 alcd_algo_dir_name = comparison_parameters["processing"][alcd_algo]["sub_dir"]
92 96  
93   - # get all the statistics files
  97 + # get all the statistics files from all the directories
94 98 all_stats_files_temp = get_all_stats_files(paths_configuration, locations, alcd_algo_dir_name)
95 99 all_stats_files = list(all_stats_files_temp)
96 100 if len(excluded) != 0:
... ... @@ -99,7 +103,7 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
99 103 if e in op.basename(a):
100 104 all_stats_files.remove(a)
101 105  
102   -
  106 + # Filters the stats files
103 107 maja_stats = [f for f in all_stats_files if 'maja_ini' in op.basename(f)]
104 108 maja_eroded_stats = [f for f in all_stats_files if 'maja_erode_' in op.basename(f)]
105 109  
... ... @@ -110,7 +114,6 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
110 114 fmask_dilated_stats = [f for f in all_stats_files if 'fmask_dilat_' in op.basename(f)]
111 115  
112 116  
113   -
114 117 if alcd_algo == "alcd_dilat":
115 118 maja_stats = maja_stats
116 119 sen2cor_stats = sen2cor_dilated_stats
... ... @@ -131,41 +134,36 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
131 134 full_fmask_txt = []
132 135  
133 136 for scene_name in all_scenes_names:
  137 + # go through all the potential scenes
134 138 scene_maja_stats = [m for m in maja_stats if scene_name in m]
135 139 scene_sen2cor_stats = [m for m in sen2cor_stats if scene_name in m]
136 140 scene_fmask_stats = [m for m in fmask_stats if scene_name in m]
137 141  
138   -
139   -
140 142 location = dict_scenes[scene_name]['location']
141 143 clear_date = dict_scenes[scene_name]['clear_date']
142 144 cloudy_date = dict_scenes[scene_name]['cloudy_date']
143 145 tile = dict_scenes[scene_name]['tile']
144 146 scene_id = dict_scenes[scene_name]['scene_id']
145 147  
146   -
  148 + # try, to avoid having an empty list
147 149 try:
148 150 data_maja = get_metrics_from_stats(scene_maja_stats[0])
149 151 data_maja_txt = ['{:.03f}'.format(m) for m in data_maja]
150 152 except:
151   - data_maja_txt = ['/']*4
152   -
153   -
  153 + data_maja_txt = ['/']*4
154 154 try:
155 155 data_sen2cor = get_metrics_from_stats(scene_sen2cor_stats[0])
156 156 data_sen2cor_txt = ['{:.03f}'.format(m) for m in data_sen2cor]
157 157 except:
158   - data_sen2cor_txt = ['/']*4
159   -
160   -
161   -
  158 + data_sen2cor_txt = ['/']*4
162 159 try:
163 160 data_fmask = get_metrics_from_stats(scene_fmask_stats[0])
164 161 data_fmask_txt = ['{:.03f}'.format(m) for m in data_fmask]
165 162 except:
166 163 data_fmask_txt = ['/']*4
167 164  
168   -
  165 + # for LaTeX, we put in bold the greater value between the chains
  166 + # for each metric and scene
169 167 if data_maja_txt[0] != '/':
170 168 for k in range(len(data_maja)):
171 169 if data_maja[k] >= data_sen2cor[k] and data_maja[k] >= data_fmask[k]:
... ... @@ -181,7 +179,7 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
181 179 if data_fmask[k] >= data_maja[k] and data_fmask[k] >= data_sen2cor[k]:
182 180 data_fmask_txt[k] = '\\textbf{' + data_fmask_txt[k] + '}'
183 181  
184   -
  182 + # print to be able to copy-paste from the console to a LaTeX editor
185 183 print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\R\hline'.format(scene_id, location, tile, cloudy_date, clear_date,
186 184 data_maja_txt[0], data_maja_txt[1], data_sen2cor_txt[0], data_sen2cor_txt[1], data_fmask_txt[0], data_fmask_txt[1]))
187 185  
... ... @@ -190,7 +188,7 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
190 188 full_sen2cor_txt.append(data_sen2cor_txt)
191 189 full_fmask_txt.append(data_fmask_txt)
192 190  
193   -
  191 + # save to a csv the results
194 192 now = datetime.datetime.now().strftime ("%Y%m%d_%H%M%S")
195 193 csv_out = op.join('tmp_maja_improvement', 'table_{}_cas{}_{}.csv'.format(file_suffix, str(cas_alcd), now))
196 194 print(csv_out)
... ... @@ -200,7 +198,6 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
200 198 writer.writerow(['scene', 'maja_acc_{}'.format(file_suffix), 'maja_f1_{}'.format(file_suffix), 'maja_reca_{}'.format(file_suffix), 'maja_prec_{}'.format(file_suffix),
201 199 'sen2cor_acc', 'sen2cor_f1', 'sen2cor_reca', 'sen2cor_prec', 'fmask_acc', 'fmask_f1', 'fmask_reca', 'fmask_prec'])
202 200 for k, scene_name in enumerate(all_scenes_names):
203   - #~ row = [scene_name, maja_log[0][k], maja_log[1][k], sen2cor_log[0][k], sen2cor_log[1][k], fmask_log[0][k], fmask_log[1][k]]
204 201 row = [scene_name,
205 202 full_maja_txt[k][0], full_maja_txt[k][1], full_maja_txt[k][2], full_maja_txt[k][3],
206 203 full_sen2cor_txt[k][0], full_sen2cor_txt[k][1], full_sen2cor_txt[k][2], full_sen2cor_txt[k][3],
... ... @@ -217,18 +214,8 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
217 214  
218 215  
219 216  
220   -
221   -
222   -
223   -
224   -
225   -
226   -
227   -
228   -
229 217 def plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd_algo = 'alcd_initial',
230   - grouping = 'by_chain', plot_type = 'errorbar', excluded = [], maja_erode = False, cas_alcd = None,
231   - print_all_stats = True):
  218 + grouping = 'by_chain', plot_type = 'errorbar', excluded = [], maja_erode = False, cas_alcd = None):
232 219 '''
233 220 Plot 4 metrics for all the processing chains
234 221 '''
... ... @@ -291,85 +278,9 @@ def plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd
291 278 data_maja, means_maja, stds_maja = get_data_and_means(maja_stats)
292 279 data_sen2cor, means_sen2cor, stds_sen2cor = get_data_and_means(sen2cor_stats)
293 280 data_fmask, means_fmask, stds_fmask = get_data_and_means(fmask_stats)
294   -
295   -
296   - if print_all_stats:
297   - csv_path = '/mnt/data/home/baetensl/clouds_detection_git/Various_data/all_sites_dates_w_names.csv'
298   -
299   -
300   - with open(csv_path, mode='r') as infile:
301   - reader = csv.reader(infile)
302   - reader.next()
303   - dict_scenes = {}
304   - scene_id = 1
305   - for row in reader:
306   - dict_scenes[row[0]] = {}
307   - dict_scenes[row[0]]['location'] = row[1]
308   - dict_scenes[row[0]]['clear_date'] = row[2]
309   - dict_scenes[row[0]]['cloudy_date'] = row[3]
310   - dict_scenes[row[0]]['tile'] = row[4]
311   - dict_scenes[row[0]]['scene_id'] = scene_id
312   - scene_id += 1
313   -
314   - previous_scene_id = 0
315   - for scene_nb in range(len(data_maja)):
316   - maja_acc = data_maja[scene_nb][0]
317   - sen2cor_acc = data_sen2cor[scene_nb][0]
318   - fmask_acc = data_fmask[scene_nb][0]
319   -
320   - maja_f1 = data_maja[scene_nb][1]
321   - sen2cor_f1 = data_sen2cor[scene_nb][1]
322   - fmask_f1 = data_fmask[scene_nb][1]
323   -
324   - scene_path_cutted = op.basename(maja_stats[scene_nb]).split('_')
325   - scene_name = scene_path_cutted[-2] + '_' + scene_path_cutted[-1].replace('.json', '')
326   -
327   -
328   - location = dict_scenes[scene_name]['location']
329   - clear_date = dict_scenes[scene_name]['clear_date']
330   - cloudy_date = dict_scenes[scene_name]['cloudy_date']
331   - tile = dict_scenes[scene_name]['tile']
332   - scene_id = dict_scenes[scene_name]['scene_id']
333   -
334   -
335   - for missing_scene_id in range(previous_scene_id+1, scene_id):
336   - print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\R\hline'.format(missing_scene_id, location, tile, cloudy_date,
337   - clear_date, ' / ', ' / ', ' / ', ' / ', ' / ', ' / '))
338   - previous_scene_id = scene_id
339   -
340   -
341   - #~ print(r'{} & {} & {} & {} & {} & {:.03f} & {:.03f} & {:.03f} & {:.03f} & {:.03f} & {:.03f} \\R\hline'.format((scene_nb+1), location, tile, cloudy_date,
342   - #~ clear_date, maja_acc, maja_f1, sen2cor_acc, sen2cor_f1, fmask_acc, fmask_f1))
343   -
344   - # Added to put in bold
345   - maja_acc_txt = '{:.03f}'.format(maja_acc)
346   - sen2cor_acc_txt = '{:.03f}'.format(sen2cor_acc)
347   - fmask_acc_txt = '{:.03f}'.format(fmask_acc)
348   -
349   - if maja_acc >= sen2cor_acc and maja_acc >= fmask_acc:
350   - maja_acc_txt = '\\textbf{' + maja_acc_txt + '}'
351   - if sen2cor_acc >= maja_acc and sen2cor_acc >= fmask_acc:
352   - sen2cor_acc_txt = '\\textbf{' + sen2cor_acc_txt + '}'
353   - if fmask_acc >= sen2cor_acc and fmask_acc >= maja_acc:
354   - fmask_acc_txt = '\\textbf{' + fmask_acc_txt + '}'
355   -
356   -
357   - maja_f1_txt = '{:.03f}'.format(maja_f1)
358   - sen2cor_f1_txt = '{:.03f}'.format(sen2cor_f1)
359   - fmask_f1_txt = '{:.03f}'.format(fmask_f1)
360   -
361   - if maja_f1 >= sen2cor_f1 and maja_f1 >= fmask_f1:
362   - maja_f1_txt = '\\textbf{' + maja_f1_txt + '}'
363   - if sen2cor_f1 >= maja_f1 and sen2cor_f1 >= fmask_f1:
364   - sen2cor_f1_txt = '\\textbf{' + sen2cor_f1_txt + '}'
365   - if fmask_f1 >= sen2cor_f1 and fmask_f1 >= maja_f1:
366   - fmask_f1_txt = '\\textbf{' + fmask_f1_txt + '}'
367   -
368   -
369   - print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\R\hline'.format(scene_id, location, tile, cloudy_date,
370   - clear_date, maja_acc_txt, maja_f1_txt, sen2cor_acc_txt, sen2cor_f1_txt, fmask_acc_txt, fmask_f1_txt))
371   -
372   -
  281 +
  282 +
  283 +
373 284  
374 285 fig, ax = plt.subplots()
375 286 ax.yaxis.grid(True) # put horizontal grid
... ... @@ -487,9 +398,10 @@ def plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd
487 398 for data in [data_maja, data_sen2cor, data_fmask]:
488 399 worst_accuracies_indexes = np.array([d[0] for d in data]).argsort()[:n]
489 400 worst_accuracies = [data[idx][0] for idx in worst_accuracies_indexes]
490   - print(worst_accuracies_indexes)
491   - print(worst_accuracies)
492   - print([op.basename(maja_stats[idx]) for idx in worst_accuracies_indexes])
  401 + if len(worst_accuracies_indexes) > 0:
  402 + print(worst_accuracies_indexes)
  403 + print(worst_accuracies)
  404 + print([op.basename(maja_stats[idx]) for idx in worst_accuracies_indexes])
493 405  
494 406 if cas_alcd == None:
495 407 out_fig = op.join('tmp_report', 'cas1_stats_synthesis_{}_{}.png'.format(plot_type, grouping))
... ... @@ -528,79 +440,13 @@ def main():
528 440  
529 441 file_suffix = 'HOT17'
530 442 make_table_metrics(paths_configuration, comparison_parameters, locations, alcd_algo, excluded = excluded, maja_erode = maja_erode, cas_alcd = cas_alcd, file_suffix = file_suffix)
531   -
532   - return
533   -
534   -
535   -
536   -
537   -
538   -
539   -
540   -
541   -
542   -
543   -
544   -
545   -
  443 +
  444 + grouping = 'by_stats'
  445 + plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd_algo,
  446 + grouping, plot_type = 'boxplot', excluded = excluded, maja_erode = maja_erode,
  447 + cas_alcd = cas_alcd)
546 448  
547 449  
548   -
549   -
550   - group = 'by_chain'
551   - group = 'by_stats'
552   - excluded = ['Gobabeb_20171014', 'Mongu_20171013']
553   - excluded = ['Gobabeb_20171014', 'Orleans_20180218']
554   - #~ excluded = ['Gobabeb_20171014',]
555   - excluded = []
556   - plot_type = 'boxplot'
557   - #~ plot_type = 'errorbar'
558   - #~ plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd_algo = 'alcd_initial', grouping = group, excluded = excluded, plot_type = plot_type)
559   -
560   -
561   -
562   - cas_alcd = 3
563   - maja = []
564   - sen2cor = []
565   - fmask = []
566   -
567   - for cas_alcd in [2]:
568   - #~ for cas_alcd in [1]:
569   - if cas_alcd == 1:
570   - alcd_algo = 'alcd_initial'
571   - maja_erode = False
572   - elif cas_alcd == 2:
573   - alcd_algo = 'alcd_dilat'
574   - maja_erode = False
575   - elif cas_alcd == 3:
576   - alcd_algo = 'alcd_initial'
577   - maja_erode = True
578   -
579   - means_maja, means_sen2cor, means_fmask = plot_all_metrics(paths_configuration, comparison_parameters, locations, alcd_algo, grouping = group, excluded = excluded, plot_type = plot_type, maja_erode = maja_erode, cas_alcd = cas_alcd)
580   -
581   - maja.append(means_maja)
582   - sen2cor.append(means_sen2cor)
583   - fmask.append(means_fmask)
584   -
585   - return
586   - # Print the mans
587   - chains = ['MAJA', 'Sen2Cor', 'Fmask']
588   - ref = ['I', 'D', 'I', 'I', 'D', 'I', 'I', 'D', 'I']
589   - comp = ['I', 'I', 'E', 'I', 'D', 'I', 'I', 'D', 'I']
590   - k = 0
591   - ref_num = 0
592   - for means in [maja, sen2cor, fmask]:
593   - cas = 0
594   - for m in means:
595   - cas += 1
596   - chain = ''
597   - if cas == 1:
598   - chain = '\multirow{3}{*}{' + chains[k] + '}'
599   - print(r"{} & {:.0f} & {} & {} & {:.03f} & {:.03f} & {:.03f} & {:.03f} \\".format(chain, cas, ref[ref_num], comp[ref_num], m[0], m[1], m[2], m[3]))
600   - ref_num += 1
601   - print('\hline')
602   - k += 1
603   -
604 450 if __name__ == '__main__':
605 451 main()
606 452  
... ...