Commit 5e637af92b00b5190082e53773a94466fdc9bc80

Authored by Louis Baetens
1 parent 09687d28
Exists in master

PCC: add the logging of erros

.gitignore
... ... @@ -3,6 +3,7 @@
3 3 */tmp/*
4 4 */tmp_report/*
5 5 */tmp_report*/*
  6 +tmp_*/
6 7 tmp/
7 8 Data_ALCD
8 9 Data_PCC
... ...
ALCD/color_tables/confidence_enhanced_byte.qml 0 → 100644
... ... @@ -0,0 +1,23 @@
  1 +<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
  2 +<qgis version="2.14.9-Essen" minimumScale="0" maximumScale="1e+08" hasScaleBasedVisibilityFlag="0">
  3 + <pipe>
  4 + <rasterrenderer opacity="1" alphaBand="-1" classificationMax="1" classificationMinMaxOrigin="User" band="1" classificationMin="0" type="singlebandpseudocolor">
  5 + <rasterTransparency/>
  6 + <rastershader>
  7 + <colorrampshader colorRampType="DISCRETE" clip="0">
  8 + <item alpha="255" value="0" label="Entrée de palette de couleurs personnalisée" color="#000000"/>
  9 + <item alpha="255" value="128" label="Entrée de palette de couleurs personnalisée" color="#ff0000"/>
  10 + <item alpha="255" value="153" label="Entrée de palette de couleurs personnalisée" color="#ff6400"/>
  11 + <item alpha="255" value="180" label="Entrée de palette de couleurs personnalisée" color="#ffc800"/>
  12 + <item alpha="255" value="205" label="Entrée de palette de couleurs personnalisée" color="#ffff00"/>
  13 + <item alpha="255" value="230" label="Entrée de palette de couleurs personnalisée" color="#ffff86"/>
  14 + <item alpha="0" value="256" label="Entrée de palette de couleurs personnalisée" color="#ff0000"/>
  15 + </colorrampshader>
  16 + </rastershader>
  17 + </rasterrenderer>
  18 + <brightnesscontrast brightness="0" contrast="0"/>
  19 + <huesaturation colorizeGreen="128" colorizeOn="0" colorizeRed="255" colorizeBlue="128" grayscaleMode="0" saturation="0" colorizeStrength="100"/>
  20 + <rasterresampler maxOversampling="2"/>
  21 + </pipe>
  22 + <blendMode>0</blendMode>
  23 +</qgis>
... ...
PCC/all_run_pcc.py
... ... @@ -17,12 +17,20 @@ import comparison
17 17 import metrics_grapher
18 18 import png_converter
19 19  
  20 +import logging
  21 +logging.basicConfig(filename='log/pcc_run.log',level=logging.ERROR, format='%(asctime)s %(message)s')
  22 +
20 23  
21 24 def create_directories(comparison_parameters):
22 25 '''
23 26 Create the directories for the code to work with
24 27 '''
25 28 print(" Creation of the directories")
  29 + for to_check in ['tmp', 'log']:
  30 + if not os.path.exists(to_check):
  31 + os.makedirs(to_check)
  32 + print(to_check + ' created')
  33 +
26 34 main_dir = comparison_parameters["user_choices"]["main_dir"]
27 35  
28 36 directories = ['', 'Multi_classif', 'Binary_classif',
... ... @@ -172,12 +180,18 @@ def run_all(part, location=None, current_date=None, masks_already_computed = Fal
172 180 # ----- Graph the metrics obtained
173 181 if 'i' in alcd_ref:
174 182 algo = 'alcd_initial'
175   - metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  183 + try:
  184 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  185 + except:
  186 + logging.exception('')
176 187 #~ for processing_chain in ['maja', 'sen2cor', 'fmask']:
177 188 #~ pixels_features_analysis.plot_confusion_repartition(comparison_parameters, processing_chain)
178 189 if 'd' in alcd_ref:
179 190 algo = 'alcd_dilat'
180   - metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  191 + try:
  192 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  193 + except:
  194 + logging.exception('')
181 195 #~ for processing_chain in ['maja', 'sen2cor', 'fmask']:
182 196 #~ pixels_features_analysis.plot_confusion_repartition(comparison_parameters, processing_chain)
183 197  
... ... @@ -216,9 +230,15 @@ def main():
216 230 location = results.location
217 231 current_date = results.current_date
218 232 alcd_ref = results.alcd_ref
219   - binary_only = str2bool(results.binary_only)
  233 +
220 234 if alcd_ref == None:
221 235 alcd_ref = 'id'
  236 +
  237 +
  238 + if results.binary_only == None:
  239 + binary_only = False
  240 + else :
  241 + binary_only = str2bool(results.binary_only)
222 242  
223 243 if results.masks_already_computed == None:
224 244 masks_already_computed = False
... ...
PCC/comparison.py
... ... @@ -8,7 +8,8 @@ import numpy as np
8 8 from PIL import Image
9 9 import glob
10 10 import json
11   -
  11 +import logging
  12 +logging.basicConfig(filename='log/pcc_run.log',level=logging.ERROR, format='%(asctime)s %(message)s')
12 13  
13 14  
14 15 def compute_comparative_image(reference_tif, to_compare_tif, out_tif, binary = True):
... ... @@ -187,12 +188,14 @@ def compare_all(comparison_parameters, binary_classif = True, reference_algo = &#39;
187 188 to_compare_tifs = [op.join(in_mask_dir, tif_name) for tif_name in all_masks_names]
188 189 difference_tifs = [op.join(out_mask_dir, 'diff_' + tif_name) for tif_name in all_masks_names]
189 190 for k in range(len(to_compare_tifs)):
190   - print('compute_comparative_image')
191   - compute_comparative_image(reference_tif, to_compare_tifs[k], difference_tifs[k], binary = binary_classif)
192   - print('compute_confusion_matrix')
193   - confusion_matrix_file = op.join(main_dir, 'Statistics', reference_sub_dir, ('stats_' + op.basename(to_compare_tifs[k])[0:-4] + '.json'))
194   - compute_confusion_matrix(difference_tifs[k], binary = binary_classif, save_file = confusion_matrix_file)
195   -
  191 + try:
  192 + compute_comparative_image(reference_tif, to_compare_tifs[k], difference_tifs[k], binary = binary_classif)
  193 + confusion_matrix_file = op.join(main_dir, 'Statistics', reference_sub_dir, ('stats_' + op.basename(to_compare_tifs[k])[0:-4] + '.json'))
  194 + compute_confusion_matrix(difference_tifs[k], binary = binary_classif, save_file = confusion_matrix_file)
  195 + except:
  196 + pass
  197 +
  198 +
196 199 def main():
197 200 comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
198 201 compare_all(comparison_parameters, binary_classif = True, reference_algo = 'alcd_initial')
... ...
PCC/find_chain_directory_paths.py
... ... @@ -37,6 +37,22 @@ def get_all_dates(location, processing_chain=&#39;maja&#39;, display = True):
37 37 return valid_dates
38 38  
39 39  
  40 +def check_existing_date(location, date_to_verify, processing_chain='maja', display = True):
  41 + '''
  42 + Check if a date is valid for a chain
  43 + '''
  44 + all_valid_dates = get_all_dates(location, processing_chain, display = False)
  45 + if date_to_verify in all_valid_dates:
  46 + date_is_valid = True
  47 + else:
  48 + date_is_valid = False
  49 +
  50 + if display:
  51 + print(date_is_valid)
  52 + return date_is_valid
  53 +
  54 +
  55 +
40 56  
41 57 def get_processing_dir(location, date_string, processing_chain='maja', display = True):
42 58 '''
... ... @@ -144,9 +160,11 @@ def get_mask_path(location, date_string, processing_chain=&#39;maja&#39;, mask_type=&#39;clo
144 160  
145 161  
146 162 def main():
147   - location = 'Arles'
148   - date_string = '20171002'
149   - get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = True)
  163 + location = 'Gobabeb'
  164 + date_string = '20180209'
  165 + check_existing_date(location, date_string, processing_chain='sen2cor', display = True)
  166 +
  167 + #~ get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = True)
150 168 return
151 169  
152 170  
... ...
PCC/log/pcc_run.log 0 → 100644
... ... @@ -0,0 +1,134 @@
  1 +2018-10-02 10:44:30,443
  2 +Traceback (most recent call last):
  3 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 566, in convert_all_masks
  4 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = True)
  5 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 263, in convert_mask_maja
  6 + convert_mask_maja_clouds(maja_cloud_mask, out_cloud, binary_only = binary_only)
  7 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 185, in convert_mask_maja_clouds
  8 + BandMathX.UpdateParameters()
  9 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 868, in UpdateParameters
  10 + def UpdateParameters(self): return _otbApplication.Application_UpdateParameters(self)
  11 +RuntimeError: Cannot open image /mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/Orleans_31UDP_20170819/Original_data/maja_original_cld.tif. The file does not exist.
  12 +2018-10-02 10:44:51,900
  13 +Traceback (most recent call last):
  14 + File "all_run_pcc.py", line 184, in run_all
  15 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  16 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 31, in plot_all_metrics
  17 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  18 +IndexError: list index out of range
  19 +2018-10-02 10:44:51,901
  20 +Traceback (most recent call last):
  21 + File "all_run_pcc.py", line 192, in run_all
  22 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  23 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 35, in plot_all_metrics
  24 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  25 +IndexError: list index out of range
  26 +2018-10-02 10:44:52,751
  27 +Traceback (most recent call last):
  28 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 579, in convert_all_masks
  29 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = False)
  30 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 297, in convert_mask_maja
  31 + BandMathX.ExecuteAndWriteOutput()
  32 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 870, in ExecuteAndWriteOutput
  33 + def ExecuteAndWriteOutput(self): return _otbApplication.Application_ExecuteAndWriteOutput(self)
  34 +RuntimeError: Exception thrown in otbApplication Application_ExecuteAndWriteOutput: /mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/include/ITK-4.12/itkImageToImageFilter.hxx:241:
  35 +itk::ERROR: BandMathXImageFilter(0x2f91400): Inputs do not occupy the same physical space!
  36 +InputImage Origin: [6.0003000e+05, 7.2000100e+06], InputImage_1 Origin: [3.9999000e+05, 5.3999700e+06]
  37 + Tolerance: 6.0000000e-05
  38 +
  39 +2018-10-02 10:45:13,917
  40 +Traceback (most recent call last):
  41 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 566, in convert_all_masks
  42 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = True)
  43 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 263, in convert_mask_maja
  44 + convert_mask_maja_clouds(maja_cloud_mask, out_cloud, binary_only = binary_only)
  45 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 185, in convert_mask_maja_clouds
  46 + BandMathX.UpdateParameters()
  47 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 868, in UpdateParameters
  48 + def UpdateParameters(self): return _otbApplication.Application_UpdateParameters(self)
  49 +RuntimeError: Cannot open image /mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/Gobabeb_33KWP_20180209/Original_data/maja_original_cld.tif. The file does not exist.
  50 +2018-10-02 10:45:28,421
  51 +Traceback (most recent call last):
  52 + File "all_run_pcc.py", line 184, in run_all
  53 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  54 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 31, in plot_all_metrics
  55 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  56 +IndexError: list index out of range
  57 +2018-10-02 10:45:28,422
  58 +Traceback (most recent call last):
  59 + File "all_run_pcc.py", line 192, in run_all
  60 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  61 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 35, in plot_all_metrics
  62 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  63 +IndexError: list index out of range
  64 +2018-10-02 10:46:28,590
  65 +Traceback (most recent call last):
  66 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 589, in convert_all_masks
  67 + convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = True)
  68 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 354, in convert_mask_sen2cor
  69 + BandMathX.UpdateParameters()
  70 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 868, in UpdateParameters
  71 + def UpdateParameters(self): return _otbApplication.Application_UpdateParameters(self)
  72 +RuntimeError: Cannot open image /mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/Munich_32UPU_20180422/Original_data/sen2cor_original_scl.tif. The file does not exist.
  73 +2018-10-02 10:46:42,556
  74 +Traceback (most recent call last):
  75 + File "all_run_pcc.py", line 184, in run_all
  76 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  77 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 32, in plot_all_metrics
  78 + sen2cor_stats = glob.glob(op.join(stats_dir, '*binary*sen2cor_ini*'))[0]
  79 +IndexError: list index out of range
  80 +2018-10-02 10:46:42,557
  81 +Traceback (most recent call last):
  82 + File "all_run_pcc.py", line 192, in run_all
  83 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  84 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 36, in plot_all_metrics
  85 + sen2cor_stats = glob.glob(op.join(stats_dir, '*binary*sen2cor_dilat*'))[0]
  86 +IndexError: list index out of range
  87 +2018-10-02 10:47:36,370
  88 +Traceback (most recent call last):
  89 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 566, in convert_all_masks
  90 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = True)
  91 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 263, in convert_mask_maja
  92 + convert_mask_maja_clouds(maja_cloud_mask, out_cloud, binary_only = binary_only)
  93 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 185, in convert_mask_maja_clouds
  94 + BandMathX.UpdateParameters()
  95 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 868, in UpdateParameters
  96 + def UpdateParameters(self): return _otbApplication.Application_UpdateParameters(self)
  97 +RuntimeError: Cannot open image /mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/RailroadValley_11SPC_20170501/Original_data/maja_original_cld.tif. The file does not exist.
  98 +2018-10-02 10:47:48,911
  99 +Traceback (most recent call last):
  100 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 589, in convert_all_masks
  101 + convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = True)
  102 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/masks_conversion.py", line 354, in convert_mask_sen2cor
  103 + BandMathX.UpdateParameters()
  104 + File "/mnt/data/home/vincenta/OTB_test/otb_superbuild/otb_superbuild-6.5-Release-install/lib/otb/python/otbApplication.py", line 868, in UpdateParameters
  105 + def UpdateParameters(self): return _otbApplication.Application_UpdateParameters(self)
  106 +RuntimeError: Cannot open image /mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/Munich_32UPU_20180424/Original_data/sen2cor_original_scl.tif. The file does not exist.
  107 +2018-10-02 10:47:55,957
  108 +Traceback (most recent call last):
  109 + File "all_run_pcc.py", line 184, in run_all
  110 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  111 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 31, in plot_all_metrics
  112 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  113 +IndexError: list index out of range
  114 +2018-10-02 10:47:55,958
  115 +Traceback (most recent call last):
  116 + File "all_run_pcc.py", line 192, in run_all
  117 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  118 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 35, in plot_all_metrics
  119 + maja_stats = glob.glob(op.join(stats_dir, '*binary*maja_ini*'))[0]
  120 +IndexError: list index out of range
  121 +2018-10-02 10:47:59,644
  122 +Traceback (most recent call last):
  123 + File "all_run_pcc.py", line 184, in run_all
  124 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  125 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 32, in plot_all_metrics
  126 + sen2cor_stats = glob.glob(op.join(stats_dir, '*binary*sen2cor_ini*'))[0]
  127 +IndexError: list index out of range
  128 +2018-10-02 10:47:59,645
  129 +Traceback (most recent call last):
  130 + File "all_run_pcc.py", line 192, in run_all
  131 + metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
  132 + File "/mnt/data/home/baetensl/clouds_detection_git/PCC/metrics_grapher.py", line 36, in plot_all_metrics
  133 + sen2cor_stats = glob.glob(op.join(stats_dir, '*binary*sen2cor_dilat*'))[0]
  134 +IndexError: list index out of range
... ...
PCC/masks_conversion.py
1 1 #!/usr/bin/python
2 2 # -*- coding: utf-8 -*-
3 3  
  4 +import os
4 5 import os.path as op
5 6 import otbApplication
6 7 import shutil
7 8 import time
8 9 import subprocess
9   -import alcd_labellisation_posttreatment
10 10 import json
  11 +import shutil
  12 +import logging
  13 +logging.basicConfig(filename='log/pcc_run.log',level=logging.ERROR, format='%(asctime)s %(message)s')
  14 +
  15 +import alcd_labellisation_posttreatment
  16 +
  17 +def resize_band(in_tif, out_tif, pixelresX, pixelresY):
  18 + '''
  19 + Resize a band with the given resolution (in meters)
  20 + '''
  21 +
  22 + # create a tmp tif, as gdal_translate can not overwrite the src
  23 + current_time = str(time.time()).replace('.', '')
  24 + tmp_tif = op.abspath(op.join('tmp', 'superimpose{}.tif'.format(current_time)))
  25 + build_warp = 'gdalwarp -tr {} {} {} {} '.format(pixelresX, pixelresY, in_tif, tmp_tif)
  26 + subprocess.call(build_warp, shell=True)
  27 + # copy the tmp to the src one, to erase it
  28 + shutil.copy(tmp_tif, out_tif)
  29 +
  30 +
  31 +
11 32  
12 33 def binar2decimal(nb):
13 34 '''
... ... @@ -449,7 +470,7 @@ def superimpose_realign(reference_tif, in_tif, out_tif):
449 470 for to_translate in [reference_tif, in_tif, out_tif]:
450 471 # create a tmp tif, as gdal_translate can not overwrite the src
451 472 current_time = str(time.time()).replace('.', '')
452   - tmp_tif = op.abspath(op.join('tmp', 'superimpose{}.tif'.format(current_time)))
  473 + tmp_tif = op.abspath(op.join('tmp', 'superimpose_{}.tif'.format(current_time)))
453 474 translate = 'gdal_translate -of GTiff '+ to_translate + ' '+ tmp_tif
454 475 subprocess.call(translate, shell=True)
455 476 # copy the tmp to the src one, to erase it
... ... @@ -490,19 +511,28 @@ def convert_all_masks(comparison_parameters, masks_already_computed = False, bin
490 511 fmask_cloud_mask_original = op.join(original_dir, comparison_parameters["processing"]["fmask"]["cloud_mask"])
491 512 alcd_mask_original = op.join(original_dir, comparison_parameters["processing"]["alcd_initial"]["cloud_mask"])
492 513  
493   - if desired_resolution == '60':
494   - reference_tif = alcd_mask_original
495   - to_realign = [maja_cloud_mask_original, maja_geo_mask_original,
496   - sen2cor_cloud_mask_original, fmask_cloud_mask_original]
497   - elif desired_resolution == '20':
498   - reference_tif = maja_cloud_mask_original
499   - to_realign = [alcd_mask_original, sen2cor_cloud_mask_original,
500   - fmask_cloud_mask_original, alcd_mask_original]
501   - else:
502   - raise Exception
503   -
  514 + # Resamples and superimpose
  515 + resize_band(alcd_mask_original, alcd_mask_original, desired_resolution, desired_resolution)
  516 +
  517 + reference_tif = alcd_mask_original
  518 + to_realign = [maja_cloud_mask_original, maja_geo_mask_original,
  519 + sen2cor_cloud_mask_original, fmask_cloud_mask_original]
  520 +
504 521 for tif_to_realign in to_realign:
505   - superimpose_realign(reference_tif, tif_to_realign, tif_to_realign)
  522 + try:
  523 + superimpose_realign(reference_tif, tif_to_realign, tif_to_realign)
  524 + except:
  525 + print('Error with {}'.format(tif_to_realign))
  526 +
  527 +
  528 + working_resolution = float(desired_resolution)
  529 + erosion_radius_meters = float(comparison_parameters["alcd_output"]["erosion_radius_meters"])
  530 + dilation_radius_meters = float(comparison_parameters["alcd_output"]["dilation_radius_meters"])
  531 +
  532 + erosion_radius_pixels = int(erosion_radius_meters/working_resolution)
  533 + dilation_radius_pixels = int(dilation_radius_meters/working_resolution)
  534 +
  535 +
506 536  
507 537  
508 538 # Convert the masks
... ... @@ -515,7 +545,8 @@ def convert_all_masks(comparison_parameters, masks_already_computed = False, bin
515 545 # dilated
516 546 alcd_initial_binary_tif = out_tif_alcd
517 547 dilated_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["alcd_dilat"]["prefix"] + suffix + '.tif')
518   - radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
  548 + #~ radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
  549 + radius = dilation_radius_pixels
519 550 alcd_labellisation_posttreatment.binary_label_dilatation(alcd_initial_binary_tif, dilated_binary, radius)
520 551  
521 552 #multi-class
... ... @@ -528,63 +559,73 @@ def convert_all_masks(comparison_parameters, masks_already_computed = False, bin
528 559 # recompute them, to save time
529 560 if not masks_already_computed:
530 561 # --- MAJA
531   - #binary
532   - out_name = 'binary_' + comparison_parameters["processing"]["maja"]["prefix"] + suffix + '.tif'
533   - out_tif_maja = op.join(binary_classif_dir, out_name)
534   - convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = True)
535   -
536   - #eroded
537   - maja_initial_binary_tif = out_tif_maja
538   - eroded_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["maja_erode"]["prefix"] + suffix + '.tif')
539   - radius = int(comparison_parameters["alcd_output"]["erosion_radius"][desired_resolution])
540   - alcd_labellisation_posttreatment.binary_label_erosion(maja_initial_binary_tif, eroded_binary, radius)
541   -
542   - #multi-class
543   - if not binary_only:
544   - out_name = 'multi_' + comparison_parameters["processing"]["maja"]["prefix"] + suffix + '.tif'
545   - out_tif_maja = op.join(multi_classif_dir, out_name)
546   - convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = False)
547   -
  562 + try:
  563 + #binary
  564 + out_name = 'binary_' + comparison_parameters["processing"]["maja"]["prefix"] + suffix + '.tif'
  565 + out_tif_maja = op.join(binary_classif_dir, out_name)
  566 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = True)
  567 +
  568 + #eroded
  569 + maja_initial_binary_tif = out_tif_maja
  570 + eroded_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["maja_erode"]["prefix"] + suffix + '.tif')
  571 + #~ radius = int(comparison_parameters["alcd_output"]["erosion_radius"][desired_resolution])
  572 + radius = erosion_radius_pixels
  573 + alcd_labellisation_posttreatment.binary_label_erosion(maja_initial_binary_tif, eroded_binary, radius)
548 574  
  575 + #multi-class
  576 + if not binary_only:
  577 + out_name = 'multi_' + comparison_parameters["processing"]["maja"]["prefix"] + suffix + '.tif'
  578 + out_tif_maja = op.join(multi_classif_dir, out_name)
  579 + convert_mask_maja(maja_cloud_mask_original, maja_geo_mask_original, out_tif_maja, binary_only = False)
  580 + except:
  581 + logging.exception('')
549 582  
  583 +
550 584 # --- SEN2COR
551   - #binary
552   - out_name = 'binary_' + comparison_parameters["processing"]["sen2cor"]["prefix"] + suffix + '.tif'
553   - out_tif_sen2cor = op.join(binary_classif_dir, out_name)
554   - convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = True)
555   -
556   - #dilated
557   - initial_binary_tif = out_tif_sen2cor
558   - dilated_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["sen2cor_dilat"]["prefix"] + suffix + '.tif')
559   - radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
560   - alcd_labellisation_posttreatment.binary_label_dilatation(initial_binary_tif, dilated_binary, radius, regularization = False)
561   -
562   - #multi-class
563   - if not binary_only:
564   - out_name = 'multi_' + comparison_parameters["processing"]["sen2cor"]["prefix"] + suffix + '.tif'
565   - out_tif_sen2cor = op.join(multi_classif_dir, out_name)
566   - convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = False)
  585 + try:
  586 + #binary
  587 + out_name = 'binary_' + comparison_parameters["processing"]["sen2cor"]["prefix"] + suffix + '.tif'
  588 + out_tif_sen2cor = op.join(binary_classif_dir, out_name)
  589 + convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = True)
  590 +
  591 + #dilated
  592 + initial_binary_tif = out_tif_sen2cor
  593 + dilated_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["sen2cor_dilat"]["prefix"] + suffix + '.tif')
  594 + #~ radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
  595 + radius = dilation_radius_pixels
  596 + alcd_labellisation_posttreatment.binary_label_dilatation(initial_binary_tif, dilated_binary, radius, regularization = False)
  597 +
  598 + #multi-class
  599 + if not binary_only:
  600 + out_name = 'multi_' + comparison_parameters["processing"]["sen2cor"]["prefix"] + suffix + '.tif'
  601 + out_tif_sen2cor = op.join(multi_classif_dir, out_name)
  602 + convert_mask_sen2cor(sen2cor_cloud_mask_original, out_tif_sen2cor, binary_only = False)
  603 + except:
  604 + logging.exception('')
567 605  
568 606  
569 607  
570 608 # --- FMASK
571   - #binary
572   - out_name = 'binary_' + comparison_parameters["processing"]["fmask"]["prefix"] + suffix + '.tif'
573   - out_tif_fmask = op.join(binary_classif_dir, out_name)
574   - convert_mask_fmask(fmask_cloud_mask_original, out_tif_fmask, binary_only = True)
575   -
576   - #dilated
577   - initial_binary_tif = out_tif_fmask
578   - dilated_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["fmask_dilat"]["prefix"] + suffix + '.tif')
579   - radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
580   - alcd_labellisation_posttreatment.binary_label_dilatation(initial_binary_tif, dilated_binary, radius, regularization = False)
581   -
582   - #multi-class
583   - if not binary_only:
584   - out_name = 'multi_' + comparison_parameters["processing"]["fmask"]["prefix"] + suffix + '.tif'
585   - out_tif_fmask = op.join(multi_classif_dir, out_name)
586   - convert_mask_fmask(fmask_cloud_mask_original, out_tif_fmask, binary_only = False)
587   -
  609 + try:
  610 + #binary
  611 + out_name = 'binary_' + comparison_parameters["processing"]["fmask"]["prefix"] + suffix + '.tif'
  612 + out_tif_fmask = op.join(binary_classif_dir, out_name)
  613 + convert_mask_fmask(fmask_cloud_mask_original, out_tif_fmask, binary_only = True)
  614 +
  615 + #dilated
  616 + initial_binary_tif = out_tif_fmask
  617 + dilated_binary = op.join(binary_classif_dir, 'binary_' + comparison_parameters["processing"]["fmask_dilat"]["prefix"] + suffix + '.tif')
  618 + #~ radius = int(comparison_parameters["alcd_output"]["dilatation_radius"][desired_resolution])
  619 + radius = dilation_radius_pixels
  620 + alcd_labellisation_posttreatment.binary_label_dilatation(initial_binary_tif, dilated_binary, radius, regularization = False)
  621 +
  622 + #multi-class
  623 + if not binary_only:
  624 + out_name = 'multi_' + comparison_parameters["processing"]["fmask"]["prefix"] + suffix + '.tif'
  625 + out_tif_fmask = op.join(multi_classif_dir, out_name)
  626 + convert_mask_fmask(fmask_cloud_mask_original, out_tif_fmask, binary_only = False)
  627 + except:
  628 + logging.exception('')
588 629  
589 630  
590 631  
... ...
PCC/parameters_files/comparison_parameters.json
... ... @@ -5,12 +5,14 @@
5 5 "20": "24",
6 6 "60": "8"
7 7 },
  8 + "dilation_radius_meters": "480",
8 9 "erosion_radius": {
9 10 "20": "24",
10 11 "60": "8"
11 12 },
  13 + "erosion_radius_meters": "480",
12 14 "labeled_img_name": "labeled_img_regular.tif",
13   - "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Arles_31TFJ_20171002",
  15 + "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/RailroadValley_11SPC_20170501",
14 16 "resolution": "60"
15 17 },
16 18 "processing": {
... ... @@ -68,10 +70,10 @@
68 70 }
69 71 },
70 72 "user_choices": {
71   - "current_date": "20171002",
72   - "location": "Arles",
73   - "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/Arles_31TFJ_20171002",
74   - "raw_img": "Arles_bands.tif",
75   - "tile": "31TFJ"
  73 + "current_date": "20170501",
  74 + "location": "RailroadValley",
  75 + "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017/RailroadValley_11SPC_20170501",
  76 + "raw_img": "RailroadValley_bands.tif",
  77 + "tile": "11SPC"
76 78 }
77 79 }
78 80 \ No newline at end of file
... ...
PCC/statistics_synthesis.py
... ... @@ -63,7 +63,7 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
63 63 excluded = [], maja_erode = False, cas_alcd = None,
64 64 print_all_stats = True, file_suffix = ''):
65 65 '''
66   - Plot 4 metrics for all the processing chains
  66 + Make tables with the results of the chains on each scene
67 67 '''
68 68  
69 69  
... ... @@ -184,7 +184,7 @@ def make_table_metrics(paths_configuration, comparison_parameters, locations, al
184 184  
185 185 print(r'{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} \\R\hline'.format(scene_id, location, tile, cloudy_date, clear_date,
186 186 data_maja_txt[0], data_maja_txt[1], data_sen2cor_txt[0], data_sen2cor_txt[1], data_fmask_txt[0], data_fmask_txt[1]))
187   -
  187 +
188 188  
189 189 full_maja_txt.append(data_maja_txt)
190 190 full_sen2cor_txt.append(data_sen2cor_txt)
... ...
Tools/alcd_launch/launch_alcd_L1C.sh 0 → 100644
... ... @@ -0,0 +1,56 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  2 +python all_run_alcd.py -l Arles -d 20170917 -c 20170920 -f true -s 2
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  4 +python all_run_alcd.py -l Arles -d 20171002 -c 20171005 -f true -s 2
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  6 +python all_run_alcd.py -l Arles -d 20171221 -c 20171224 -f true -s 2
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  8 +python all_run_alcd.py -l Gobabeb -d 20161221 -c 20161231 -f true -s 2
  9 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  10 +python all_run_alcd.py -l Gobabeb -d 20170909 -c 20170914 -f true -s 2
  11 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  12 +python all_run_alcd.py -l Gobabeb -d 20171014 -c 20171019 -f true -s 2
  13 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  14 +python all_run_alcd.py -l Gobabeb -d 20180209 -c 20180214 -f true -s 2
  15 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  16 +python all_run_alcd.py -l Ispra -d 20170815 -c 20170820 -f true -s 2
  17 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  18 +python all_run_alcd.py -l Ispra -d 20171009 -c 20171014 -f true -s 2
  19 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  20 +python all_run_alcd.py -l Ispra -d 20171111 -c 20171116 -f true -s 2
  21 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  22 +python all_run_alcd.py -l Marrakech -d 20160417 -c 20160427 -f true -s 2
  23 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  24 +python all_run_alcd.py -l Marrakech -d 20170621 -c 20170701 -f true -s 2
  25 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  26 +python all_run_alcd.py -l Marrakech -d 20171218 -c 20171223 -f true -s 2
  27 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  28 +python all_run_alcd.py -l Mongu -d 20161112 -c 20161202 -f true -s 2
  29 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  30 +python all_run_alcd.py -l Mongu -d 20170804 -c 20170809 -f true -s 2
  31 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  32 +python all_run_alcd.py -l Mongu -d 20171013 -c 20171018 -f true -s 2
  33 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  34 +python all_run_alcd.py -l Orleans -d 20170516 -c 20170526 -f true -s 2
  35 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  36 +python all_run_alcd.py -l Orleans -d 20170819 -c 20170829 -f true -s 2
  37 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  38 +python all_run_alcd.py -l Orleans -d 20180218 -c 20180225 -f true -s 2
  39 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  40 +python all_run_alcd.py -l Pretoria -d 20170313 -c 20170323 -f true -s 2
  41 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  42 +python all_run_alcd.py -l Pretoria -d 20170820 -c 20170825 -f true -s 2
  43 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  44 +python all_run_alcd.py -l Pretoria -d 20171014 -c 20171019 -f true -s 2
  45 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  46 +python all_run_alcd.py -l Pretoria -d 20171213 -c 20171218 -f true -s 2
  47 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  48 +python all_run_alcd.py -l RailroadValley -d 20170501 -c 20170504 -f true -s 2
  49 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  50 +python all_run_alcd.py -l RailroadValley -d 20170827 -c 20170901 -f true -s 2
  51 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  52 +python all_run_alcd.py -l RailroadValley -d 20180213 -c 20180218 -f true -s 2
  53 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  54 +python all_run_alcd.py -l Munich -d 20180422 -c 20180427 -f true -s 2
  55 +cd /mnt/data/home/baetensl/clouds_detection_git/ALCD
  56 +python all_run_alcd.py -l Munich -d 20180424 -c 20180427 -f true -s 2
... ...
Tools/create_pcc_launch.py
... ... @@ -66,13 +66,13 @@ def create_alcd_launch(locations, clear_dates, cloudy_dates, out_file):
66 66  
67 67  
68 68 def main():
69   - csv_path = '/mnt/data/home/baetensl/clouds_detection_git/Various_data/all_26_sites.csv'
  69 + csv_path = '/mnt/data/home/baetensl/clouds_detection_git/Various_data/all_28_sites.csv'
70 70 locations, clear_dates, cloudy_dates = open_csv(csv_path)
71 71  
72   - out_file_dir = '/mnt/data/home/baetensl/clouds_detection_git/Tools/tmp/launch_pcc'
  72 + out_file_dir = '/mnt/data/home/baetensl/clouds_detection_git/Tools/pcc_launch/launch_pcc'
73 73 create_pcc_launch(locations, cloudy_dates, out_file_dir)
74 74  
75   - out_file = '/mnt/data/home/baetensl/clouds_detection_git/Tools/tmp/launch_alcd_L1C.sh'
  75 + out_file = '/mnt/data/home/baetensl/clouds_detection_git/Tools/alcd_launch/launch_alcd_L1C.sh'
76 76 create_alcd_launch(locations, clear_dates, cloudy_dates, out_file)
77 77  
78 78  
... ...
Tools/export_data_alcd_publi.py 0 → 100644
... ... @@ -0,0 +1,183 @@
  1 +#!/usr/bin/python
  2 +# -*- coding: utf-8 -*-
  3 +
  4 +import sys
  5 +import os
  6 +import os.path as op
  7 +import glob
  8 +import numpy as np
  9 +import json
  10 +import csv
  11 +import shutil
  12 +import subprocess
  13 +
  14 +def open_csv(csv_path):
  15 + locations = []
  16 + clear_dates = []
  17 + cloudy_dates = []
  18 + with open(csv_path, 'rb') as csvfile:
  19 + spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
  20 + spamreader.next()
  21 + for row in spamreader:
  22 + locations.append(row[1])
  23 + clear_dates.append(row[2])
  24 + cloudy_dates.append(row[3])
  25 + print(locations)
  26 + print(clear_dates)
  27 + print(cloudy_dates)
  28 + return locations, clear_dates, cloudy_dates
  29 +
  30 +
  31 +def get_alcd_dirs(src_data_alcd):
  32 + return glob.glob(op.join(src_data_alcd, '*'))
  33 +
  34 +
  35 +def get_product_name(location, date, display = True):
  36 + '''
  37 + Get the full name of a L1C scene from its location and date
  38 + '''
  39 + paths_configuration = json.load(open(op.join('..', 'paths_configuration.json')))
  40 + if location[0].isdigit():
  41 + L1C_dir = '/mnt/data/home/baetensl/Hollstein/L1C_products'
  42 + else:
  43 + L1C_dir = paths_configuration["global_chains_paths"]["L1C"]
  44 +
  45 + location_dir = op.join(L1C_dir, location)
  46 + with_date = glob.glob(op.join(location_dir, 'S2*_{}*.SAFE'.format(date)))[0]
  47 +
  48 +
  49 + product_name = op.basename(with_date).replace('.SAFE', '')
  50 + if display:
  51 + print(product_name)
  52 + return product_name
  53 +
  54 +
  55 +def create_dir(directory):
  56 + if not op.exists(directory):
  57 + os.makedirs(directory)
  58 +
  59 +def get_location_dates_from_global_parameters(global_parameters_path):
  60 + global_parameters = json.load(open(global_parameters_path))
  61 +
  62 + location = global_parameters["user_choices"]["location"]
  63 + cloudy_date = global_parameters["user_choices"]["current_date"]
  64 + clear_date = global_parameters["user_choices"]["clear_date"]
  65 +
  66 + return location, cloudy_date, clear_date
  67 +
  68 +
  69 +
  70 +
  71 +def convert_tif(in_tif, out_tif):
  72 + # Create the JPG
  73 + translate = 'gdal_translate -scale 0 1 -ot Byte '+ in_tif + ' '+ out_tif
  74 + subprocess.call(translate, shell=True)
  75 +
  76 +def create_json(out_json, in_global_parameters_path):
  77 + global_parameters = json.load(open(in_global_parameters_path))
  78 +
  79 + location = global_parameters["user_choices"]["location"]
  80 + cloudy_date = global_parameters["user_choices"]["current_date"]
  81 + clear_date = global_parameters["user_choices"]["clear_date"]
  82 + tile = global_parameters["user_choices"]["tile"]
  83 + cloudy_product_name = get_product_name(location, cloudy_date, display = False)
  84 + clear_product_name = get_product_name(location, clear_date, display = False)
  85 +
  86 + data = {}
  87 + data["location"] = location
  88 + data["cloudy_date"] = cloudy_date
  89 + data["clear_date"] = clear_date
  90 + data["tile"] = tile
  91 + data["cloudy_product_name"] = cloudy_product_name
  92 + data["clear_product_name"] = clear_product_name
  93 +
  94 + # Save our changes to JSON file
  95 + jsonFile = open(out_json, "w+")
  96 + jsonFile.write(json.dumps(data, indent=3, sort_keys=True))
  97 + jsonFile.close()
  98 +
  99 +
  100 +
  101 +def copy_all_data(src_data_alcd, dst_data_alcd):
  102 + # Create the destination dir
  103 + create_dir(dst_data_alcd)
  104 +
  105 + # Get the src folders paths
  106 + src_folders = get_alcd_dirs(src_data_alcd)
  107 +
  108 + for src_dir in src_folders:
  109 + print('\n{}'.format(op.basename(src_dir)))
  110 + global_parameters_path = op.join(src_dir, 'In_data', 'used_global_parameters.json')
  111 + location, cloudy_date, clear_date = get_location_dates_from_global_parameters(global_parameters_path)
  112 +
  113 + # create the output dir
  114 + cloudy_product_name = get_product_name(location, cloudy_date, display = False)
  115 + if not location[0].isdigit():
  116 + main_out_dir = op.join(dst_data_alcd, '26scenes', cloudy_product_name)
  117 + else:
  118 + main_out_dir = op.join(dst_data_alcd, 'Hollstein', cloudy_product_name)
  119 +
  120 + create_dir(main_out_dir)
  121 +
  122 + sub_dirs = ['Classification', 'Samples', 'Statistics']
  123 + for sub in sub_dirs:
  124 + create_dir(op.join(main_out_dir, sub))
  125 +
  126 + in_files = []
  127 + out_files = []
  128 +
  129 + # Copy the classification
  130 + in_files.append(op.join(src_dir, 'Out', 'labeled_img_regular.tif'))
  131 + out_files.append(op.join(main_out_dir, 'Classification', 'classification_map.tif'))
  132 +
  133 + # Copy the contours
  134 + in_files.append(op.join(src_dir, 'Out', 'contours_superposition.png'))
  135 + out_files.append(op.join(main_out_dir, 'Classification', 'contours.png'))
  136 +
  137 + # Convert the confidence_enhanced to Byte, and copy it
  138 + in_confidence = op.join(src_dir, 'Out', 'confidence_enhanced.tif')
  139 + out_confidence = op.join(main_out_dir, 'Classification', 'confidence_enhanced.tif')
  140 + convert_tif(in_confidence, out_confidence)
  141 +
  142 + # Copy the Samples for each class
  143 + class_names = ['land', 'water', 'snow', 'no_data', 'low_clouds', 'high_clouds', 'clouds_shadows']
  144 + extensions = ['.dbf', '.prj', '.shp', '.shx']
  145 + for class_name in class_names:
  146 + for extension in extensions:
  147 + file_name = class_name + extension
  148 + in_files.append(op.join(src_dir, 'In_data', 'Masks', file_name))
  149 + out_files.append(op.join(main_out_dir, 'Samples', file_name))
  150 +
  151 +
  152 + # Copy the Kfold stats (all the stats and mean)
  153 + in_files.append(op.join(src_dir, 'Statistics', 'k_fold_summary.json'))
  154 + out_files.append(op.join(main_out_dir, 'Statistics', 'k_fold_summary.json'))
  155 +
  156 + # Actually copy the files
  157 + for k in range(len(in_files)):
  158 + shutil.copy(in_files[k], out_files[k])
  159 +
  160 + # Create a txt file with the used_parameters.json
  161 + out_json = op.join(main_out_dir, 'Classification', 'used_parameters.json')
  162 + create_json(out_json, global_parameters_path)
  163 +
  164 +
  165 +
  166 +
  167 +
  168 +
  169 +
  170 +def main():
  171 + src_data_alcd = '/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD'
  172 + dst_data_alcd = '/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD_publication'
  173 +
  174 + copy_all_data(src_data_alcd, dst_data_alcd)
  175 + return
  176 +
  177 + location = 'Arles'
  178 + date = '20170917'
  179 + get_product_name(location, date)
  180 +
  181 +
  182 +if __name__ == '__main__':
  183 + main()
... ...
Tools/pcc_launch/launch_pcc_0.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Arles -d 20170917
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Arles -d 20171002
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Arles -d 20171221
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Gobabeb -d 20161221
... ...
Tools/pcc_launch/launch_pcc_1.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Gobabeb -d 20170909
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Gobabeb -d 20171014
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Gobabeb -d 20180209
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Ispra -d 20170815
... ...
Tools/pcc_launch/launch_pcc_2.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Ispra -d 20171009
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Ispra -d 20171111
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Marrakech -d 20160417
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Marrakech -d 20170621
... ...
Tools/pcc_launch/launch_pcc_3.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Marrakech -d 20171218
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Mongu -d 20161112
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Mongu -d 20170804
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Mongu -d 20171013
... ...
Tools/pcc_launch/launch_pcc_4.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Orleans -d 20170516
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Orleans -d 20170819
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Orleans -d 20180218
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Pretoria -d 20170313
... ...
Tools/pcc_launch/launch_pcc_5.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l Pretoria -d 20170820
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l Pretoria -d 20171014
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Pretoria -d 20171213
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l RailroadValley -d 20170501
... ...
Tools/pcc_launch/launch_pcc_6.sh 0 → 100644
... ... @@ -0,0 +1,8 @@
  1 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  2 +python all_run_pcc.py -l RailroadValley -d 20170827
  3 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  4 +python all_run_pcc.py -l RailroadValley -d 20180213
  5 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  6 +python all_run_pcc.py -l Munich -d 20180422
  7 +cd /mnt/data/home/baetensl/clouds_detection_git/PCC
  8 +python all_run_pcc.py -l Munich -d 20180424
... ...
Tools/pcc_launch/launch_pcc_7.sh 0 → 100644
paths_configuration.json
... ... @@ -12,7 +12,7 @@
12 12 },
13 13 "data_paths": {
14 14 "data_alcd": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD",
15   - "data_pcc": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT018"
  15 + "data_pcc": "/mnt/data/home/baetensl/clouds_detection_git/Data_PCC_HOT017"
16 16 },
17 17 "tile_location": {
18 18 "Orleans": "31UDP",
... ...