all_run_pcc.py 11.4 KB
#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys 
import os
import os.path as op
import json
import shutil
import glob
import argparse
import time

import masks_conversion
import find_chain_directory_paths
import comparison
import metrics_grapher
import png_converter

import logging
logging.basicConfig(filename='log/pcc_run.log',level=logging.ERROR, format='%(asctime)s %(message)s')


def create_directories(comparison_parameters):
    '''
    Create the directories for the code to work with
    '''    
    print("  Creation of the directories")
    for to_check in ['tmp', 'log']:
        if not os.path.exists(to_check):
            os.makedirs(to_check)
            print(to_check + ' created')


    main_dir = comparison_parameters["user_choices"]["main_dir"]
    directories = ['', 'Multi_classif', 'Binary_classif', 
    'Multi_difference', 'Binary_difference', 'Original_data', 
    'Intermediate', 'Statistics', 'Out', op.join('Statistics', 'features')]
    for sub_dir in directories:
        current_dir = op.join(main_dir, sub_dir)
        if not os.path.exists(current_dir):
            os.makedirs(current_dir)
            print(current_dir + ' created')
            
    sub_dirs = [comparison_parameters["processing"]["alcd_initial"]["sub_dir"],
    comparison_parameters["processing"]["alcd_dilat"]["sub_dir"]]
    
    root_dirs = ['Multi_difference', 'Binary_difference',
    'Statistics', 'Out']
    for sub in sub_dirs:
        for root in root_dirs:
            current_dir = op.join(main_dir, root, sub)
            if not os.path.exists(current_dir):
                os.makedirs(current_dir)
                print(current_dir + ' created')            
    print('Done')


def initialization_comparison_parameters(main_dir, location, current_date):
    ''' To initialize the path and name in the JSON file
    Must be done at the very beggining
    '''
    paths_configuration = json.load(open(op.join('..', 'paths_configuration.json')))
    
    json_path = op.join('parameters_files','comparison_parameters.json')
    jsonFile = open(json_path, "r") # Open the JSON file for reading
    data = json.load(jsonFile) # Read the JSON into the buffer
    jsonFile.close() # Close the JSON file

    # Working with buffered content
    data["user_choices"]["main_dir"] = main_dir
    data["user_choices"]["current_date"] = current_date
    data["user_choices"]["location"] = location
    data["user_choices"]["raw_img"] = location + '_bands.tif'

    tile = paths_configuration["tile_location"][location]
    data["user_choices"]["tile"] = tile
    
    data["alcd_output"]["main_dir"] = op.join(paths_configuration["data_paths"]["data_alcd"], (location + '_' + tile + '_' + current_date))

    # Save our changes to JSON file
    jsonFile = open(json_path, "w+")
    jsonFile.write(json.dumps(data, indent=3, sort_keys=True))
    jsonFile.close()
    


    
def run_all(part, location=None, current_date=None, masks_already_computed = False, 
            comparison_parameters = None, alcd_ref = 'idc', binary_only = False):
    paths_configuration = json.load(open(op.join('..', 'paths_configuration.json')))
    Data_PCC_dir = paths_configuration["data_paths"]["data_pcc"]    
    Data_ALCD_dir = paths_configuration["data_paths"]["data_alcd"]    

    tile = paths_configuration["tile_location"][location]
    
    if part == 1:
        # ----- Create the directories and general structure
        main_dir = op.join(Data_PCC_dir, (location + '_' + tile + '_' + current_date))
        initialization_comparison_parameters(main_dir, location, current_date)
        
        # Load the parameters
        comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
        create_directories(comparison_parameters)
        
        # Copy the global_parameters files to save it
        src = op.join('parameters_files','comparison_parameters.json')
        dst = op.join(Data_PCC_dir, comparison_parameters["user_choices"]["main_dir"], 'used_comparison_parameters.json')
        shutil.copyfile(src, dst)
        return comparison_parameters
        

    else:
        if comparison_parameters == None:
            comparison_parameters = json.load(open(op.join('parameters_files','comparison_parameters.json')))
        main_dir = comparison_parameters["user_choices"]["main_dir"]
        location = comparison_parameters["user_choices"]["location"]
        date_string = comparison_parameters["user_choices"]["current_date"]
        tile = comparison_parameters["user_choices"]["tile"]
        
    
    if part == 2:    
        # ----- Copy the original results from the different algorithms
        if not masks_already_computed:
            try:
                # sen2cor
                sen2cor_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='sen2cor', mask_type='cloud', display = False)
                shutil.copy(sen2cor_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["sen2cor"]["cloud_mask"]))
            except:
                logging.exception('')
                
            try:        
                # maja
                maja_cloud_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='maja', mask_type='cloud', display = False)
                shutil.copy(maja_cloud_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["maja"]["cloud_mask"]))
                maja_geo_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='maja', mask_type='geo', display = False)
                shutil.copy(maja_geo_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["maja"]["geo_mask"]))
            except:
                logging.exception('')
                
            try:    
                # fmask
                if comparison_parameters["processing"]["fmask"]["version"] == "3":
                    fmask_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='fmask3', mask_type='cloud', display = False)                
                else:
                    fmask_mask = find_chain_directory_paths.get_mask_path(location, date_string, processing_chain='fmask', mask_type='cloud', display = False)
                shutil.copy(fmask_mask, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["fmask"]["cloud_mask"]))
            except:
                logging.exception('')
                
                                        
        # original labeled ALCD
        try:
            alcd_classif_name = comparison_parameters["alcd_output"]["labeled_img_name"]
            alcd_output = op.join(Data_ALCD_dir, (location + '_' + tile + '_' + date_string), 'Out', alcd_classif_name)
            shutil.copy(alcd_output, op.join(main_dir, 'Original_data', comparison_parameters["processing"]["alcd_initial"]["cloud_mask"]))
        except:
            logging.exception('')


    elif part == 3:
        # ----- Convert the different masks
        masks_conversion.convert_all_masks(comparison_parameters, 
        masks_already_computed = masks_already_computed, 
        binary_only = binary_only)
                    
    elif part == 4:
        # ----- Compute the difference between the masks and the statistics
        if 'i' in alcd_ref:
            algo = 'alcd_initial'
            comparison.compare_all(comparison_parameters, binary_classif = True, reference_algo = algo)
            if not binary_only:
                comparison.compare_all(comparison_parameters, binary_classif = False, reference_algo = algo)

        if 'd' in alcd_ref:
            algo = 'alcd_dilat'
            comparison.compare_all(comparison_parameters, binary_classif = True, reference_algo = algo)


    elif part == 5:
        # ----- Graph the metrics obtained
        if 'i' in alcd_ref:
            algo = 'alcd_initial'        
            try:
                metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
            except:
                logging.exception('')
 
        if 'd' in alcd_ref:
            algo = 'alcd_dilat'        
            try:
                metrics_grapher.plot_all_metrics(comparison_parameters, reference_algo = algo)
            except:
                logging.exception('')

                                                
    elif part == 6:
        # ----- Post treat the difference masks
        png_converter.post_treatment(comparison_parameters)
        # clean the .aux.xml files
        main_dir = comparison_parameters["user_choices"]["main_dir"]
        for root, dirnames, filenames in os.walk(main_dir):
            to_delete = [op.join(root, name) for name in filenames if '.aux.xml' in name]
            for file_to_delete in to_delete:
                os.remove(file_to_delete)
        
    
    
    
def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')    
    
def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-l', action='store', default=None, dest='location', help='Location (e.g. Orleans)')
    parser.add_argument('-d', action='store', default=None, dest='current_date', help='The desired date to process (e.g. 20170702)')
    parser.add_argument('-m', action='store', default=None, dest='masks_already_computed', help='Bool, if the masks have already been computed')
    parser.add_argument('-r', action='store', default=None, dest='alcd_ref', help='Which chains you want to compare to. Can contain i (initial), d (dilation). e.g. id for initial and dilation')
    parser.add_argument('-b', action='store', default=None, dest='binary_only', help='If you only want to compute the binary difference. False will compute the multi-class difference also')
    
    results = parser.parse_args()    
    
    location = results.location
    current_date = results.current_date
    alcd_ref = results.alcd_ref

    if alcd_ref == None:
        alcd_ref = 'id'

    if results.binary_only == None:
        binary_only = False
    else :
        binary_only = str2bool(results.binary_only)
        
    if results.masks_already_computed == None:
        masks_already_computed = False
    else :
        masks_already_computed = str2bool(results.masks_already_computed)
    
    
    if location == None or current_date == None:
        location = 'Orleans'
        current_date = '20170516'

    
    comparison_parameters = run_all(1, location, current_date, masks_already_computed, alcd_ref)
    # Saving the comparison parameters in a variable allows to run
    # multiple PCC in parallel on different locations and dates at the same time
    run_all(2, location, current_date, masks_already_computed, comparison_parameters, alcd_ref)
    run_all(3, location, current_date, masks_already_computed, comparison_parameters, alcd_ref, binary_only = binary_only)
    run_all(4, location, current_date, masks_already_computed, comparison_parameters, alcd_ref, binary_only = binary_only)
    run_all(5, location, current_date, masks_already_computed, comparison_parameters, alcd_ref)
    run_all(6, location, current_date, masks_already_computed, comparison_parameters, alcd_ref)
        
if __name__=='__main__':
    start_time = time.time()
    main()
    total_time = time.time() - start_time
    minutes = int((float(total_time)/60))
    seconds = int((total_time-minutes*60))
    print("--- {} min {} s ---".format(minutes, seconds))