Commit 72ea5fb38dd7b793896a1ae8b9a1765d0d26712e

Authored by Louis Baetens
1 parent 87993c57
Exists in master

small update

ALCD/metrics_exploitation.py
... ... @@ -406,9 +406,9 @@ def get_all_locations_dates(csv_file):
406 406 reader = csv.reader(f)
407 407 headers = next(reader, None) #skip the headers
408 408 for row in reader:
409   - locations.append(row[0])
410   - clear_dates.append(row[1])
411   - cloudy_dates.append(row[2])
  409 + locations.append(row[1])
  410 + clear_dates.append(row[2])
  411 + cloudy_dates.append(row[3])
412 412 return locations, clear_dates, cloudy_dates
413 413  
414 414  
... ... @@ -501,7 +501,7 @@ def plot_statistics_all_sites():
501 501  
502 502 def plot_mean_statistics_all_sites(plot_both = True):
503 503 global_parameters = json.load(open(op.join('parameters_files','global_parameters.json')))
504   - csv_file = op.join('/mnt/data/home/baetensl/clouds_detection_git/Various_data', 'all_sites_dates.csv')
  504 + csv_file = op.join('/mnt/data/home/baetensl/clouds_detection_git/Tools', 'all_scenes.csv')
505 505 locations, _, dates = get_all_locations_dates(csv_file)
506 506  
507 507 all_metrics = []
... ... @@ -520,7 +520,8 @@ def plot_mean_statistics_all_sites(plot_both = True):
520 520  
521 521 accuracies.append([[m[0] for m in temp_metrics]])
522 522 f1scores.append([[m[1] for m in temp_metrics]])
523   - scenes_names.append('{}_{}'.format(location, date))
  523 + location_txt = location.replace('Alta_Floresta_Brazil', 'AltaFloresta')
  524 + scenes_names.append('{}_{}'.format(location_txt, date))
524 525  
525 526  
526 527 except:
... ... @@ -576,9 +577,13 @@ def plot_mean_statistics_all_sites(plot_both = True):
576 577 plt.show(block=False)
577 578  
578 579 out_fig = op.join('tmp_report', 'kfold_synthese_mean.png')
579   - print('Figure saved in {}'.format(out_fig))
580 580 plt.savefig(out_fig, bbox_inches='tight')
581 581 plt.close()
  582 + print('Figure saved in {}'.format(out_fig))
  583 +
  584 + print('Accuracy: mean = {:.5f}, std = {:.5f}'.format(np.mean(acc_means), np.mean(acc_stds)))
  585 + print('F1-score: mean = {:.5f}, std = {:.5f}'.format(np.mean(f1_means), np.mean(f1_stds)))
  586 +
582 587 return
583 588  
584 589  
... ... @@ -588,7 +593,8 @@ def plot_mean_statistics_all_sites(plot_both = True):
588 593  
589 594 def main():
590 595 global_parameters = json.load(open(op.join('parameters_files','global_parameters.json')))
591   - retrieve_Kfold_data(global_parameters, metrics_plotting = True)
  596 + #~ retrieve_Kfold_data(global_parameters, metrics_plotting = True)
  597 + plot_mean_statistics_all_sites(plot_both = True)
592 598  
593 599  
594 600  
... ...
PCC/parameters_files/comparison_parameters.json
... ... @@ -6,13 +6,13 @@
6 6 "resolution": "60"
7 7 },
8 8 "automatically_generated": {
9   - "alcd_main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Arles_31TFJ_20171002",
10   - "current_date": "20171002",
11   - "location": "Arles",
12   - "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/HOT016/Arles_31TFJ_20171002",
13   - "raw_img": "Arles_bands.tif",
14   - "sub_data_dir": "HOT016",
15   - "tile": "31TFJ"
  9 + "alcd_main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Alta_Floresta_Brazil_21LWK_20180813",
  10 + "current_date": "20180813",
  11 + "location": "Alta_Floresta_Brazil",
  12 + "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC/HOT016_11_10/Alta_Floresta_Brazil_21LWK_20180813",
  13 + "raw_img": "Alta_Floresta_Brazil_bands.tif",
  14 + "sub_data_dir": "HOT016_11_10",
  15 + "tile": "21LWK"
16 16 },
17 17 "maja_parameters": {
18 18 "maja_dir_name": "MAJA_3_1_S2AS2B_HOT016",
... ... @@ -66,4 +66,4 @@
66 66 "prefix": "sen2cor_dilat"
67 67 }
68 68 }
69 69 -}
  70 +}
70 71 \ No newline at end of file
... ...
PCC/statistics_synthesis.py
... ... @@ -203,7 +203,8 @@ def make_table_metrics(paths_configuration, comparison_parameters, scenes_csv,
203 203 data_fmask_txt[k] = '\\textbf{' + data_fmask_txt[k] + '}'
204 204  
205 205 # print to be able to copy-paste from the console to a LaTeX editor
206   - print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\R\hline'.format(scene_id, location, tile, cloudy_date, clear_date,
  206 + location_txt = location.replace('Alta_Floresta_Brazil', 'AltaFloresta')
  207 + print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\ \hline'.format(scene_id, location_txt, tile, cloudy_date, clear_date,
207 208 data_maja_txt[0], data_maja_txt[1], data_sen2cor_txt[0], data_sen2cor_txt[1], data_fmask_txt[0], data_fmask_txt[1]))
208 209  
209 210  
... ... @@ -213,7 +214,8 @@ def make_table_metrics(paths_configuration, comparison_parameters, scenes_csv,
213 214  
214 215 # save to a csv the results
215 216 now = datetime.datetime.now().strftime ("%Y%m%d_%H%M%S")
216   - csv_out = op.join('tmp_maja_improvement', 'table_{}_cas{}_{}.csv'.format(file_suffix, str(cas_alcd), now))
  217 + #~ csv_out = op.join('tmp_maja_improvement', 'table_{}_cas{}_{}.csv'.format(file_suffix, str(cas_alcd), now))
  218 + csv_out = op.join('tmp_maja_improvement', 'table_{}_cas{}.csv'.format(pcc_sub_dir, str(cas_alcd)))
217 219 print(csv_out)
218 220  
219 221 with open(csv_out, "wb") as f:
... ... @@ -322,11 +324,14 @@ def plot_all_metrics_simplified(data_maja, data_sen2cor, data_fmask, out_fig_nam
322 324 #~ plt.legend(custom_lines, all_legend_names, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
323 325 plt.legend(custom_lines, all_legend_names, loc=3, ncol=2)
324 326  
325   - if len(set(nb_of_scenes_per_chain)) == 0:
326   - plt.title('Statistics over {} images'.format(nb_of_scenes_per_chain))
327   - else:
328   - plt.title('Statistics over {}/{}/{} images'.format(nb_of_scenes_per_chain[0],
329   - nb_of_scenes_per_chain[1], nb_of_scenes_per_chain[2]))
  327 + title = False
  328 +
  329 + if title:
  330 + if len(set(nb_of_scenes_per_chain)) == 0:
  331 + plt.title('Statistics over {} images'.format(nb_of_scenes_per_chain))
  332 + else:
  333 + plt.title('Statistics over {}/{}/{} images'.format(nb_of_scenes_per_chain[0],
  334 + nb_of_scenes_per_chain[1], nb_of_scenes_per_chain[2]))
330 335  
331 336 #~ plt.show(block = False)
332 337  
... ...
Tools/pcc_launch/launch_pcc_all.sh
1 1 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
2   -python all_run_pcc.py -l Arles -d 20170917 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  2 +python all_run_pcc.py -l Arles -d 20170917 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
3 3 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
4   -python all_run_pcc.py -l Arles -d 20171002 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  4 +python all_run_pcc.py -l Arles -d 20171002 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
5 5 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
6   -python all_run_pcc.py -l Arles -d 20171221 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  6 +python all_run_pcc.py -l Arles -d 20171221 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
7 7 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
8   -python all_run_pcc.py -l Gobabeb -d 20161221 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  8 +python all_run_pcc.py -l Gobabeb -d 20161221 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
9 9 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
10   -python all_run_pcc.py -l Gobabeb -d 20170909 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  10 +python all_run_pcc.py -l Gobabeb -d 20170909 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
11 11 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
12   -python all_run_pcc.py -l Gobabeb -d 20171014 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  12 +python all_run_pcc.py -l Gobabeb -d 20171014 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
13 13 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
14   -python all_run_pcc.py -l Gobabeb -d 20180209 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  14 +python all_run_pcc.py -l Gobabeb -d 20180209 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
15 15 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
16   -python all_run_pcc.py -l Ispra -d 20170815 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  16 +python all_run_pcc.py -l Ispra -d 20170815 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
17 17 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
18   -python all_run_pcc.py -l Ispra -d 20171009 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  18 +python all_run_pcc.py -l Ispra -d 20171009 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
19 19 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
20   -python all_run_pcc.py -l Ispra -d 20171111 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  20 +python all_run_pcc.py -l Ispra -d 20171111 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
21 21 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
22   -python all_run_pcc.py -l Marrakech -d 20160417 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  22 +python all_run_pcc.py -l Marrakech -d 20160417 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
23 23 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
24   -python all_run_pcc.py -l Marrakech -d 20170621 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  24 +python all_run_pcc.py -l Marrakech -d 20170621 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
25 25 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
26   -python all_run_pcc.py -l Marrakech -d 20171218 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  26 +python all_run_pcc.py -l Marrakech -d 20171218 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
27 27 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
28   -python all_run_pcc.py -l Mongu -d 20161112 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  28 +python all_run_pcc.py -l Mongu -d 20161112 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
29 29 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
30   -python all_run_pcc.py -l Mongu -d 20170804 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  30 +python all_run_pcc.py -l Mongu -d 20170804 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
31 31 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
32   -python all_run_pcc.py -l Mongu -d 20171013 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  32 +python all_run_pcc.py -l Mongu -d 20171013 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
33 33 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
34   -python all_run_pcc.py -l Orleans -d 20170516 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  34 +python all_run_pcc.py -l Orleans -d 20170516 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
35 35 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
36   -python all_run_pcc.py -l Orleans -d 20170819 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  36 +python all_run_pcc.py -l Orleans -d 20170819 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
37 37 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
38   -python all_run_pcc.py -l Orleans -d 20180218 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  38 +python all_run_pcc.py -l Orleans -d 20180218 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
39 39 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
40   -python all_run_pcc.py -l Pretoria -d 20170313 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  40 +python all_run_pcc.py -l Pretoria -d 20170313 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
41 41 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
42   -python all_run_pcc.py -l Pretoria -d 20170820 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  42 +python all_run_pcc.py -l Pretoria -d 20170820 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
43 43 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
44   -python all_run_pcc.py -l Pretoria -d 20171014 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  44 +python all_run_pcc.py -l Pretoria -d 20171014 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
45 45 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
46   -python all_run_pcc.py -l Pretoria -d 20171213 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  46 +python all_run_pcc.py -l Pretoria -d 20171213 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
47 47 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
48   -python all_run_pcc.py -l RailroadValley -d 20170501 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  48 +python all_run_pcc.py -l RailroadValley -d 20170501 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
49 49 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
50   -python all_run_pcc.py -l RailroadValley -d 20170827 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  50 +python all_run_pcc.py -l RailroadValley -d 20170827 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
51 51 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
52   -python all_run_pcc.py -l RailroadValley -d 20180213 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  52 +python all_run_pcc.py -l RailroadValley -d 20180213 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
53 53 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
54   -python all_run_pcc.py -l Munich -d 20180422 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  54 +python all_run_pcc.py -l Munich -d 20180422 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
55 55 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
56   -python all_run_pcc.py -l Munich -d 20180424 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  56 +python all_run_pcc.py -l Munich -d 20180424 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
57 57 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
58   -python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180505 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  58 +python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180505 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
59 59 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
60   -python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180609 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  60 +python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180609 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
61 61 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
62   -python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180714 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  62 +python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180714 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
63 63 cd /mnt/data/home/baetensl/clouds_detection_git/PCC
64   -python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180813 -s HOT016_final -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
  64 +python all_run_pcc.py -l Alta_Floresta_Brazil -d 20180813 -s HOT016_11_10 -mver 3 -mdir MAJA_3_1_S2AS2B_HOT016
... ...