Olivier Hagolle / LANDSAT-Download

Download as
Browse Code »

Commit 979bb1754a8987bac137046aa693590db268e087

Authored by Olivier Hagolle
1 parent 88834cda
Exists in master

bug fixes for list mode and proxy, and integration of jake-brinkmann/LANDSAT-Download improvements

Showing 1 changed file with 71 additions and 46 deletions   Show diff stats
download_landsat_scene.py
  Diff comments   View file @979bb17
1 1 #! /usr/bin/env python
2 2 # -*- coding: iso-8859-1 -*-
3   -import os,sys,math,urllib2,urllib
  3 +
  4 +"""
  5 + Landsat Data download from earth explorer
  6 + Incorporates jake-Brinkmann improvements
  7 +"""
  8 +import os,sys,math,urllib2,urllib,time
4 9 import optparse
5 10 ###########################################################################
6 11 class OptionParser (optparse.OptionParser):
... ... @@ -14,17 +19,12 @@ class OptionParser (optparse.OptionParser):
14 19  
15 20 #############################"Connection à Earth explorer sans proxy
16 21  
17   -def connect_earthexplorer_proxy(usgs,proxy):
  22 +def connect_earthexplorer_proxy(proxy_info,usgs):
18 23  
19   - proxy_info = {
20   - 'user' : proxy['account'],
21   - 'pass' : proxy['passwd'],
22   - 'host' : proxy['address'],
23   - 'port' : proxy['port'],
24   - }
  24 + print "proxy_info=", proxy_info
25 25 # contruction d'un "opener" qui utilise une connexion proxy avec autorisation
26   - proxy_support = urllib2.ProxyHandler({"http" : "http://%(user)s:%(pass)s@%(host)s:%(port)d" % proxy_info,
27   - "https" : "http://%(user)s:%(pass)s@%(host)s:%(port)d" % proxy_info})
  26 + proxy_support = urllib2.ProxyHandler({"http" : "http://%(user)s:%(pass)s@%(host)s:%(port)s" % proxy_info,
  27 + "https" : "http://%(user)s:%(pass)s@%(host)s:%(port)s" % proxy_info})
28 28 opener = urllib2.build_opener(proxy_support, urllib2.HTTPCookieProcessor)
29 29  
30 30 # installation
... ... @@ -37,9 +37,16 @@ def connect_earthexplorer_proxy(usgs,proxy):
37 37 f = opener.open('https://earthexplorer.usgs.gov/login', params)
38 38 data = f.read()
39 39 f.close()
40   -
  40 +
  41 + if data.find('You must sign in as a registered user to download data or place orders for USGS EROS products')>0 :
  42 + print "Authentification failed"
  43 + sys.exit(-1)
  44 +
  45 +
  46 +
41 47 return
42 48  
  49 +
43 50 #############################"Connection à Earth explorer sans proxy
44 51  
45 52 def connect_earthexplorer_no_proxy(usgs):
... ... @@ -53,12 +60,17 @@ def connect_earthexplorer_no_proxy(usgs):
53 60 print "Authentification failed"
54 61 sys.exit(-1)
55 62 return
  63 +#############################
56 64  
57   -#############################"pour des gros fichiers
58   -
  65 +def sizeof_fmt(num):
  66 + for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
  67 + if num < 1024.0:
  68 + return "%3.1f %s" % (num, x)
  69 + num /= 1024.0
  70 +#############################
59 71 def downloadChunks(url,rep,nom_fic):
60   - """Telecharge de gros fichiers par morceaux
61   - inspire de http://josh.gourneau.com
  72 + """ Downloads large files in pieces
  73 + inspired by http://josh.gourneau.com
62 74 """
63 75  
64 76 try:
... ... @@ -74,24 +86,36 @@ def downloadChunks(url,rep,nom_fic):
74 86 print sys.exit(-1)
75 87 total_size = int(req.info().getheader('Content-Length').strip())
76 88 if (total_size<50000):
77   - print "erreur : le fichier est trop petit pour etre une image landsat"
78   - print url
79   - sys.exit(-1)
  89 + print "Error: The file is too small to be a Landsat Image"
  90 + print url
  91 + sys.exit(-1)
80 92 print nom_fic,total_size
  93 + total_size_fmt = sizeof_fmt(total_size)
  94 +
81 95 downloaded = 0
82 96 CHUNK = 1024 * 1024 *8
83 97 with open(rep+'/'+nom_fic, 'wb') as fp:
84   - while True:
85   - chunk = req.read(CHUNK)
86   - downloaded += len(chunk)
87   - sys.stdout.write(str(math.floor((float(downloaded) / total_size) * 100 )) +'%')
88   - sys.stdout.flush()
89   - if not chunk: break
90   - fp.write(chunk)
91   - print 'fini'
  98 + start = time.clock()
  99 + print('Downloading {0} ({1}):'.format(nom_fic, total_size_fmt))
  100 + while True:
  101 + chunk = req.read(CHUNK)
  102 + downloaded += len(chunk)
  103 + done = int(50 * downloaded / total_size)
  104 + sys.stdout.write('\r[{1}{2}]{0:3.0f}% {3}ps'
  105 + .format(math.floor((float(downloaded)
  106 + / total_size) * 100),
  107 + '=' * done,
  108 + ' ' * (50 - done),
  109 + sizeof_fmt((downloaded // (time.clock() - start)) / 8)))
  110 + sys.stdout.flush()
  111 + if not chunk: break
  112 + fp.write(chunk)
92 113 except urllib2.HTTPError, e:
93   - print "HTTP Error:",e.code , url
94   - return False
  114 + if e.code == 500:
  115 + pass # File doesn't exist
  116 + else:
  117 + print "HTTP Error:", e.code , url
  118 + return False
95 119 except urllib2.URLError, e:
96 120 print "URL Error:",e.reason , url
97 121 return False
... ... @@ -99,7 +123,7 @@ def downloadChunks(url,rep,nom_fic):
99 123 return rep,nom_fic
100 124  
101 125 ######################################################################################
102   -###############""main
  126 +###############main
103 127 #################################################################################
104 128  
105 129 ################Lecture des arguments
... ... @@ -115,21 +139,23 @@ else:
115 139 usage = "usage: %prog [options] "
116 140 parser = OptionParser(usage=usage)
117 141 parser.add_option("-o", "--option", dest="option", action="store", type="choice", \
118   - help="scene ou liste", choices=['scene','liste'],default=None)
  142 + help="scene or liste", choices=['scene','liste'],default=None)
119 143 parser.add_option("-l", "--liste", dest="fic_liste", action="store", type="string", \
120   - help="nom du fichier liste",default=None)
  144 + help="liste filename",default=None)
121 145 parser.add_option("-s", "--scene", dest="scene", action="store", type="string", \
122 146 help="coordonnees WRS2 de la scene (ex 198030)", default=None)
123 147 parser.add_option("-a", "--annee", dest="annee", action="store", type="int", \
124 148 help="annee")
125 149 parser.add_option("-d", "--doy_deb", dest="doy_deb", action="store", type="int", \
126   - help="premier jour dans l'annee")
  150 + help="first landsat day of year for tile ")
127 151 parser.add_option("-f","--doy_fin", dest="doy_fin", action="store", type="int", \
128   - help="dernier jour dans l'annee")
  152 + help="last day of year for tile")
129 153 parser.add_option("-u","--usgs_passwd", dest="usgs", action="store", type="string", \
130 154 help="USGS earthexplorer account and password file")
131 155 parser.add_option("-p","--proxy_passwd", dest="proxy", action="store", type="string", \
132 156 help="Proxy account and password file")
  157 + parser.add_option("--output", dest="output", action="store", type="string", \
  158 + help="Where to download files",default='/tmp/LANDSAT')
133 159  
134 160 (options, args) = parser.parse_args()
135 161 parser.check_required("-o")
... ... @@ -139,15 +165,16 @@ else:
139 165 parser.check_required("-f")
140 166 parser.check_required("-s")
141 167 parser.check_required("-u")
  168 +
142 169 elif options.option=='liste' :
143 170 parser.check_required("-l")
144 171 parser.check_required("-u")
145 172  
146   -
147   -rep='/tmp/LANDSAT'
  173 +
  174 +rep=options.output
148 175 if not os.path.exists(rep):
149 176 os.mkdir(rep)
150   - if options.option==liste:
  177 + if options.option=='liste':
151 178 if not os.path.exists(rep+'/LISTE'):
152 179 os.mkdir(rep+'/LISTE')
153 180  
... ... @@ -168,22 +195,20 @@ except :
168 195 if options.proxy != None :
169 196 try:
170 197 f=file(options.proxy)
171   - (account,passwd)=f.readline().split(' ')
  198 + (user,passwd)=f.readline().split(' ')
172 199 if passwd.endswith('\n'):
173 200 passwd=passwd[:-1]
174   - proxy={'account':account,'passwd':passwd}
175   - address=f.readline()
176   - if address.endswith('\n'):
177   - address=address[:-1]
  201 + host=f.readline()
  202 + if host.endswith('\n'):
  203 + host=host[:-1]
178 204 port=f.readline()
179 205 if port.endswith('\n'):
180 206 port=port[:-1]
181   - proxy={'port':port}
  207 + proxy={'user':user,'pass':passwd,'host':host,'port':port}
182 208 f.close()
183 209 except :
184 210 print "error with proxy password file"
185 211 sys.exit(-3)
186   -
187 212 ############Telechargement des produits par scene
188 213 if options.option=='scene':
189 214 produit='LC8'
... ... @@ -210,11 +235,11 @@ if options.option==&#39;scene&#39;:
210 235 url="http://earthexplorer.usgs.gov/download/%s/%s/STANDARD/EE"%(repert,nom_prod)
211 236 print url
212 237 if not(os.path.exists(rep_scene+'/'+nom_prod+'.tgz')):
213   - try:
214   - if options.proxy!=None :
  238 + if options.proxy!=None :
215 239 connect_earthexplorer_proxy(proxy,usgs)
216   - else:
  240 + else:
217 241 connect_earthexplorer_no_proxy(usgs)
  242 + try:
218 243 downloadChunks(url,"%s"%rep_scene,nom_prod+'.tgz')
219 244 except TypeError:
220 245 print ' produit %s non trouve'%nom_prod
... ...