Commit af3fa1fec6fa18b985d9ac879b81cc57d2bec8ae

Authored by michel-lepage
0 parents
Exists in master

first commit

README.md 0 → 100644
  1 +++ a/README.md
... ... @@ -0,0 +1,5 @@
  1 +Codes available:
  2 +
  3 +s1_gee.js : Download of Sentinel1 GRD Time Series from Google Earth Engine (actually it works also with landsat)
  4 +s2_amazonS3.py: Download of Sentinel2 L1A tiles Time series from Amazon AWS.
  5 +modis_download.py: Download of time series of MODIS products
... ...
modis_download.py 0 → 100644
  1 +++ a/modis_download.py
... ... @@ -0,0 +1,148 @@
  1 +# -*- coding: utf-8 -*-
  2 +"""
  3 +Created on Mon Nov 24 19:48:02 2014
  4 +v1.1: 25/01/2015
  5 +
  6 +@author: Michel Le Page
  7 +
  8 +Download a time series of Modis images
  9 +inspired from https://pypi.python.org/pypi/GeobricksMODIS/0.1.4
  10 +
  11 +Parameters
  12 +test # (boolean) True to test the list of file names, False to actually make the downloading
  13 +product_name # (string) The MODIS product name (for example MOD13Q1, MYD13Q1, MOD11A1...)
  14 +product_version # (string) Most of the product are in version '5'
  15 +increment # (int) Time step of the product in days. For example MOD13Q1 has a time step of 16 days
  16 +hh # (string) The horizontal id of the tile to download, for example "18"
  17 +vv # (string the vertical id of the tile to download, for example "05"
  18 +from_year # (int) first year to download (modis data begin in 2000)
  19 +to_year # (int) last year of data to download
  20 +output_dir # (string) the directory where to download the data
  21 +day00 # (int) the first julian date to download for the first year (for example 321 or 1)
  22 +"""
  23 +from ftplib import FTP
  24 +import os
  25 +import glob
  26 +
  27 +
  28 +def modis_download(output_dir, product_name='MOD13Q1',product_version='5',increment=16,hh='18',vv='05',from_year=2000,to_year=2017,day00=1):
  29 +
  30 +
  31 +
  32 + # URL et repertoires des données MoDiS
  33 + base_url="ladsweb.nascom.nasa.gov"
  34 + data_dir="/allData/5/"
  35 + retries=1
  36 +
  37 + # ---------- that's all -----------------
  38 +
  39 + counts=[0,0,0]
  40 + local_filename=''
  41 + ftp=FTP(base_url)
  42 + ftp.login()
  43 +
  44 + for year in range(from_year,to_year):
  45 + if year==from_year:
  46 + day0=day00
  47 + #elif year==2005:
  48 + # day0=145
  49 + else:
  50 + day0=1
  51 +
  52 + for day in range(day0,367,increment):
  53 + stryear=str(year)
  54 + strday=str(day).zfill(3)
  55 + fname=output_dir+product_name+'.A'+stryear+strday+'.h'+hh+'v'+vv+'*.hdf'
  56 + aaa=glob.glob(fname)
  57 + if aaa.__len__()>0:
  58 + print(" *** Local File ",aaa[0]," already exists")
  59 + counts[1]=counts[1]+1
  60 + else:
  61 + try:
  62 + ftp=FTP(base_url)
  63 + ftp.login()
  64 + ftp.cwd(data_dir)
  65 + ftp.cwd(product_name.upper())
  66 + ftp.cwd(stryear)
  67 + ftp.cwd(strday)
  68 + #---- 7 juin 2015: MLSD does not work anymore...
  69 + #ls = [] #MLSD
  70 + #ftp.retrlines('MLSD', ls.append) #MLSD
  71 +
  72 + ls=ftp.nlst()
  73 +
  74 + #for line in ls: #MLSD
  75 + for file_name in ls:
  76 +
  77 + #start = line.index(product_name.upper()) #MLSD
  78 + #file_name = line[start:] #MLSD
  79 +
  80 + if file_name.find('h'+hh+'v'+vv)>0:
  81 + #start = line.index('Size=') #MLSD
  82 + #end = line.index(';', start) #MLSD
  83 + #size = line[start + len('Size='):end] #MLSD
  84 + #print(file_name+":"+size) #MLSD
  85 + file_path="ftp://"+base_url+data_dir
  86 + file_path += product_name.upper() + '/' + stryear + '/' + strday + '/'
  87 + file_path += file_name
  88 + #file_path += line[start:] #MLSD
  89 + h = file_name[2 + file_name.index('.h'):4 + file_name.index('.h')]
  90 + v = file_name[1 + file_name.index('v'):3 + file_name.index('v')]
  91 + #label = 'H ' + h + ', V ' + v + ' (' + str(round((float(size) / 1000000), 2)) + ' MB)' #MLSD
  92 +
  93 + local_filename=output_dir+file_name
  94 +
  95 + if os.path.exists(local_filename) is False:
  96 + print(file_path)
  97 +
  98 + if test is not True:
  99 + retry=0
  100 + ok=False
  101 + size_ftp=ftp.size(file_name)
  102 +
  103 + while retry<retries and ok==False:
  104 + with open(local_filename, 'wb') as f:
  105 + # Define the callback as a closure so it can access the opened
  106 + # file in local scope
  107 + def callback(data):
  108 + f.write(data)
  109 +
  110 + ftp.retrbinary('RETR %s' % file_name, callback)
  111 +
  112 + # test is the file is fully downloaded
  113 + f.close()
  114 +
  115 + size_downloaded=os.path.getsize(local_filename)
  116 + if size_ftp!=size_downloaded:
  117 + retry=retry+1
  118 + ok=False
  119 + os.remove(local_filename)
  120 + print("*=*=*=*=*=*= Download error! Retrying...")
  121 + else:
  122 + counts[0]=counts[0]+1
  123 + ok=True
  124 + else:
  125 + print(" *** Local File ",local_filename," already exists")
  126 + except:
  127 + print(" **** File not found or error downloading ",year,day)
  128 + counts[2]=counts[2]+1
  129 + if local_filename!='': os.remove(local_filename)
  130 + pass
  131 +
  132 + ftp.quit()
  133 +
  134 +if __name__ == "__main__":
  135 +
  136 + # produit/tuile à telecharger
  137 + test=False # True pour tester la liste des fichiers sinon False
  138 + product_name="MOD11A1" # produit modis
  139 + product_version='5'
  140 + increment=1 # pas de temps des images MODIS
  141 + hh="17" # coordonnée horizontale de la tuile
  142 + vv="05" # coordonnée vertical de la tuile
  143 + from_year=2000 # premiere annee
  144 + to_year=2017 # derniere annee
  145 + output_dir="C:\\" # repertoire de sortie
  146 + day00=65
  147 +
  148 + modis_download(output_dir, product_name,product_version,increment,hh,vv,from_year,to_year,day00)
... ...
s1_gee.js 0 → 100644
  1 +++ a/s1_gee.js
... ... @@ -0,0 +1,51 @@
  1 +/* -----------------------------------------------------------------------------------------------------------------------
  2 +Sentinel 1 download from Google Earth Engine
  3 +Author: Michel Le Page, CESBIO/IRD
  4 +Version: 0.1, Feb. 2016
  5 +
  6 +First you have to ask Google an access to Google Earth Engine, they should give you access within 1 or 2 days.
  7 +then you create your javascript code with https://code.earthengine.google.com
  8 +
  9 +The first thing is to choose the area to download: use the polygon tool to draw the area you are interested in.
  10 +it will create a new object in the code called "geometry"
  11 +
  12 +Click on run.
  13 +
  14 +It will create a graph in the console and several tasks.
  15 +Run each of those tasks manually, and it will save the images on your google drive.
  16 +Finally, download the files from google drive to your computer.
  17 +
  18 +The exported files are georeferenced subsets of each S1 image in GeoTIFF at 20m spatial resolution. The three bands are VH, VV and angle.
  19 +The projection in this code is EPSG:4326 (geographic lat-lon), you can look for other projections at http://spatialreference.org/
  20 +
  21 +A CSV table resuming what has been selected is also created.
  22 +
  23 +----------------------------------------------------------------------------------------------------------------------- */
  24 +
  25 +var start = new Date("02/05/2015"); // The first date
  26 +var end = new Date(Date.now()); // The last date
  27 +
  28 +var s1 = ee.ImageCollection('COPERNICUS/S1_GRD') // The product to download. It is the same technique for landsat L8T
  29 + .filterDate(start, end)
  30 + //.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VV'))
  31 + //.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VH'))
  32 + // Filter to get images collected in interferometric wide swath mode.
  33 + //.filter(ee.Filter.eq('instrumentMode', 'IW'))
  34 + .filterBounds(geometry);
  35 +
  36 +var count = s1.size().getInfo();
  37 +
  38 +for (var i = 0; i < count ; i++) {
  39 + var img = ee.Image(s1.toList(1, i).get(0));
  40 + var geom = img.geometry().getInfo();
  41 + Export.image(img, img.get('system:index').getInfo(), {
  42 + 'scale': 20,
  43 + 'crs': 'EPSG:4326',
  44 + 'region': geometry.toGeoJSONString()
  45 +});
  46 +}
  47 +
  48 +Export.table(s1,'Sentinel1_csv');
  49 +
  50 +print(Chart.image.seriesByRegion(s1, geometry, ee.Reducer.mean(),'VH')
  51 +.setOptions({title: 'VH mean value over time on R3'}));?
0 52 \ No newline at end of file
... ...
s2_amazonS3.py 0 → 100644
  1 +++ a/s2_amazonS3.py
... ... @@ -0,0 +1,57 @@
  1 +# -*- coding: utf-8 -*-
  2 +"""
  3 +Created on Wed Feb 24 21:22:00 2016
  4 +
  5 +@author: Michel Le Page
  6 +
  7 +This script depends on "SDK AWS pour Python (Boto3)": https://aws.amazon.com/fr/sdk-for-python/
  8 +pip install boto3
  9 +
  10 +https://github.com/sinergise/Sentinel2ProductIngestor
  11 +Each file is its own object in Amazon S3. The data are organised per tiles using the Military grid system.
  12 +The latest KML file describing available tiles is available is ESA site (https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-2/data-products),
  13 +or via direct link (https://sentinels.copernicus.eu/documents/247904/1955685/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml/ec05e22c-a2bc-4a13-9e84-02d5257b09a8).
  14 +SHP version is available here: https://github.com/justinelliotmeyers/Sentinel-2-Shapefile-Index
  15 +
  16 +"""
  17 +
  18 +import boto3
  19 +from botocore.handlers import disable_signing
  20 +import os
  21 +
  22 +# ------------------------------------------------------
  23 +# To be modified by the user:
  24 +# tiles = tiles ID as in the Kml or Shp indicated above
  25 +# years = the years to download
  26 +#
  27 +tiles=['43/Q/HV']
  28 +years=['2015','2016']
  29 +dst_dir='C:\\Users\\michel\\Documents\\SIE\\SIE_Download'
  30 +
  31 +# ------------------------------------------------------
  32 +
  33 +S2bucket2='sentinel-s2-l1c'
  34 +
  35 +s3 = boto3.resource('s3')
  36 +s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
  37 +
  38 +bucket = s3.Bucket(S2bucket2)
  39 +
  40 +for tile in tiles:
  41 + for year in years:
  42 + kk=bucket.objects.filter(Prefix='tiles/'+tile+'/'+year)
  43 +
  44 + for key in kk.all():
  45 + print(key.key,' ', key.size,' bytes')
  46 + aaa=key.key.split('/')
  47 + tileid=aaa[1]+aaa[2]+aaa[3]
  48 + ladate=aaa[4]+'_'+aaa[5]+'_'+aaa[6]
  49 + name=aaa[len(aaa)-1]
  50 + path0 = os.path.join(dst_dir, tileid)
  51 + if not(os.path.isdir(path0)):
  52 + os.mkdir(path0)
  53 + path1 = os.path.join(path0, ladate)
  54 + if not(os.path.isdir(path1)):
  55 + os.mkdir(path1)
  56 + path2 = os.path.join(path1, name)
  57 + bucket.download_file(key.key, path2)
... ...