Commit c6620c1115df7d25d293a53a14b3db9b558797ab

Authored by Louis Baetens
1 parent 3a404f33
Exists in master

Added the readme

ALCD/confidence_map_exploitation.py
... ... @@ -15,6 +15,7 @@ import numpy as np
15 15 import tempfile
16 16 from collections import defaultdict
17 17 import matplotlib.pyplot as plt
  18 +
18 19 import merge_shapefiles
19 20  
20 21  
... ...
ALCD/layers_creation.py
... ... @@ -146,9 +146,9 @@ def create_no_data_shp(global_parameters, force = False):
146 146 '''
147 147 main_dir = global_parameters["user_choices"]["main_dir"]
148 148  
149   - if op.exists(op.join(main_dir, 'In_data', 'Masks', 'no_data.shp')) and force == False:
150   - print('No_data layer already present, use -force to erase and replace')
151   - return
  149 + #~ if op.exists(op.join(main_dir, 'In_data', 'Masks', 'no_data.shp')) and force == False:
  150 + #~ print('No_data layer already present, use -force to erase and replace')
  151 + #~ return
152 152  
153 153 tmp_name = next(tempfile._get_candidate_names())
154 154 tmp_tif = op.join('tmp', 'no_data_mask_{}.tif'.format(tmp_name))
... ...
ALCD/parameters_files/global_parameters.json
... ... @@ -103,11 +103,11 @@
103 103 "training_proportion": "0.9"
104 104 },
105 105 "user_choices": {
106   - "clear_date": "20171223",
107   - "current_date": "20171218",
108   - "location": "Marrakech",
109   - "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Marrakech_29RPQ_20171218",
110   - "raw_img": "Marrakech_bands.tif",
111   - "tile": "29RPQ"
  106 + "clear_date": "20180724",
  107 + "current_date": "20180714",
  108 + "location": "Alta_Floresta_Brazil",
  109 + "main_dir": "/mnt/data/home/baetensl/clouds_detection_git/Data_ALCD/Alta_Floresta_Brazil_21LWK_20180714",
  110 + "raw_img": "Alta_Floresta_Brazil_bands.tif",
  111 + "tile": "21LWK"
112 112 }
113 113 }
114 114 \ No newline at end of file
... ...
ALCD/quicklook_generator.py
... ... @@ -27,11 +27,10 @@ def create_jpg(in_jp2s, out_jpg):
27 27 return
28 28  
29 29  
30   -def create_all_quicklook(location):
  30 +def create_all_quicklook(location, out_dir):
31 31 '''
32 32 Create all the quicklooks for a given location
33 33 '''
34   - out_dir = op.join('/mnt/data/home/baetensl/clouds_detection_git/Tools/tmp/Quicks', location)
35 34 if not os.path.exists(out_dir):
36 35 os.makedirs(out_dir)
37 36  
... ... @@ -62,21 +61,13 @@ def create_all_quicklook(location):
62 61  
63 62 def main():
64 63 locations = ['Mongu', 'Gobabeb', 'RailroadValley', 'Arles', 'Marrakech']
65   - locations = ['Italie']
66   - for location in locations:
67   - #~ location = 'Pretoria'
68   -
69   - create_all_quicklook(location)
  64 + locations = ['Alta_Floresta_Brazil']
  65 + for location in locations:
  66 + out_dir = op.join('/mnt/data/home/baetensl/clouds_detection_git/Tools/tmp/Quicks', location)
  67 + create_all_quicklook(location, out_dir)
70 68  
71 69 return
72   - R = '/mnt/data/SENTINEL2/L1C_PDGS/Mongu/S2B_MSIL1C_20171222T082329_N0206_R121_T34LGJ_20171222T105215.SAFE/GRANULE/L1C_T34LGJ_A004151_20171222T083834/IMG_DATA/T34LGJ_20171222T082329_B04.jp2'
73   - G = '/mnt/data/SENTINEL2/L1C_PDGS/Mongu/S2B_MSIL1C_20171222T082329_N0206_R121_T34LGJ_20171222T105215.SAFE/GRANULE/L1C_T34LGJ_A004151_20171222T083834/IMG_DATA/T34LGJ_20171222T082329_B03.jp2'
74   - B = '/mnt/data/SENTINEL2/L1C_PDGS/Mongu/S2B_MSIL1C_20171222T082329_N0206_R121_T34LGJ_20171222T105215.SAFE/GRANULE/L1C_T34LGJ_A004151_20171222T083834/IMG_DATA/T34LGJ_20171222T082329_B02.jp2'
75   - in_jp2s = [R,G,B]
76   -
77   - out_jpg = '/mnt/data/home/baetensl/classification_clouds/Codes/test.jpg'
78   -
79   - create_jpg(in_jp2s, out_jpg)
  70 +
80 71  
81 72 if __name__=='__main__':
82 73 main()
... ...
ALCD/readme.md 0 → 100644
... ... @@ -0,0 +1,84 @@
  1 +# Active Learning for Cloud Detection (ALCD)
  2 +
  3 +This tools allows the user to create cloud masks with an active learning process. These masks can be used as reference masks.
  4 +
  5 +This readme describes quickly the use of each python file, the general use of the tool is described in the [user manual](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD_user_manual.pdf) in the parent directory.
  6 +
  7 +## Description of each file
  8 +- [all_run_alcd.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/all_run_alcd.py)
  9 +main file, calls the other files. Run the full ALCD code.
  10 +Called by: /
  11 +
  12 +- [L1C_band_composition.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/L1C_band_composition.py)
  13 +creates the features of the full image, taking into account the original image, the cloud-free one and the DEM.
  14 +Called by: all_run_alcd.py, layers_creation.py
  15 +
  16 +- [OTB_workflow.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/OTB_workflow.py)
  17 +core of the program, relies heavily on OTB. Computes statistics on the image, trains a Random Forest, classifies the image.
  18 +Called by: all_run_alcd.py
  19 +
  20 +- [confidence_map_exploitation.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/confidence_map_exploitation.py)
  21 +creates the enhanced version of the confidence map. Also computes the evolution of the confidence and of the samples number across the iterations.
  22 +Called by: all_run_alcd.py, OTB_workflow.py
  23 +
  24 +- [contour_from_labeled.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/contour_from_labeled.py)
  25 +creates the contours of the classes from the classification map.
  26 +Called by: OTB_workflow.py
  27 +
  28 +- [expand_point_region.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/expand_point_region.py)
  29 +increases the number of samples by taking their neighborhood.
  30 +Called by: masks_preprocessing.py
  31 +
  32 +- [find_directory_names.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/find_directory_names.py)
  33 +gets the path of the L1C directory, and check if the dates given by the user are valid.
  34 +Called by: all_run_alcd.py, L1C_band_composition.py
  35 +
  36 +- [layers_creation.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/layers_creation.py)
  37 +generates empty shapefiles layer, one for each class, so the user can populate them afterwards.
  38 +Called by: all_run_alcd.py
  39 +
  40 +- [masks_preprocessing.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/masks_preprocessing.py)
  41 +calls other codes to augment the samples quantity and merge them into one shapefile, and split them between a training and validation set.
  42 +Called by: all_run_alcd.py
  43 +
  44 +- [merge_shapefiles.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/merge_shapefiles.py)
  45 +merge multiple shapefiles into one, and gives a class number to each point based on the shapefile it comes from.
  46 +Called by: confidence_map_exploitation.py, masks_preprocessing.py
  47 +
  48 +- [metrics_exploitation.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/metrics_exploitation.py)
  49 +computes various metrics, including accuracy and f1-score for the k-fold.
  50 +Called by: all_run_alcd.py
  51 +
  52 +- [quicklook_generator.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/quicklook_generator.py)
  53 +generates quicklooks of all the dates in the L1C directory for given locations.
  54 +Called by: /
  55 +
  56 +- [split_samples.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/split_samples.py)
  57 +separates point from a shapefile in two different shapefiles (training and validation sets). Also used to split into k different sets, for the k-fold cross-validation.
  58 +Called by: masks_preprocessing.py
  59 +
  60 +- [synthese_alcd_runs.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD/synthese_alcd_runs.py)
  61 +makes a synthesis of the scenes proceeded with ALCD.
  62 +Called by: /
  63 +
  64 +
  65 +## Getting Started
  66 +You need to set the parameters as described in the [user manual](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD_user_manual.pdf).
  67 +Then you can simply run
  68 +```
  69 +python all_run_alcd.py -f true -s 0 -l Arles -d 20171002 -c 20171005
  70 +```
  71 +Modify the shapefiles to populate them.
  72 +Run
  73 +```
  74 +python all_run_alcd.py -f true -s 1 -l Arles -d 20171002 -c 20171005
  75 +```
  76 +While the classification is not satisfactory, run
  77 +```
  78 +python all_run_alcd.py -f false -s 0
  79 +```
  80 +Add more points to the shapefiles.
  81 +Run
  82 +```
  83 +python all_run_alcd.py -f false -s 1
  84 +```
... ...
PCC/readme.md
... ... @@ -3,7 +3,7 @@
3 3 This tools allows the user to compare the output of cloud detection processing chains against a reference.
4 4 The reference is created with ALCD beforehand. The current supported processing chains (PC) are MAJA, Sen2Cor and Fmask
5 5  
6   -This readme describes quickly the use of each python file, the general use of the tool is described in the user manual in the parent directory.
  6 +This readme describes quickly the use of each python file, the general use of the tool is described in the [user manual](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/ALCD_user_manual.pdf) in the parent directory.
7 7  
8 8 ## Description of each file
9 9 - [all_run_pcc.py](http://tully.ups-tlse.fr/baetensl/clouds_detection/blob/master/PCC/all_run_pcc.py)
... ...
paths_configuration.json
... ... @@ -31,6 +31,7 @@
31 31 "49JHM_20160126": "49JHM",
32 32 "Marrakech":"29RPQ",
33 33 "Ispra":"32TMR",
34   - "Munich":"32UPU"
  34 + "Munich":"32UPU",
  35 + "Alta_Floresta_Brazil":"21LWK"
35 36 }
36 37 }
... ...