1 from ftplib import FTP
2 import datetime
3 from bs4 import BeautifulSoup
4 import urllib
5 from geobricks_modis.config.gaul2modis import map
6 from geobricks_modis.config.modis_config import config as conf
7
8
10 """
11 Parse the MODIS product list.
12 @return: Dictionary with the MODIS product list.
13 """
14 sock = urllib.urlopen('https://lpdaac.usgs.gov/products/modis_products_table')
15 html = sock.read()
16 sock.close()
17 soup = BeautifulSoup(html)
18 tables = soup.findAll('table')
19 table = tables[len(tables) - 1]
20 tbody = table.find('tbody')
21 trs = tbody.findAll('tr')
22 products = []
23 keys = ['code', 'platform', 'modis_data_product', 'raster_type', 'spatial_resolution', 'temporal_resolution']
24 for tr in trs:
25 p = {}
26 counter = 0
27 tds = tr.findAll('td')
28 for td in tds:
29 text = ''.join(td.find(text=True)).strip().replace('\n', '')
30 if counter == 0:
31 text = td.find('a').find(text=True)
32 p[keys[counter]] = text
33 counter += 1
34 products.append(p)
35 return products
36
37
39 """
40 List all the available MODIS products.
41 @return: An array of code/label objects.
42 """
43 if conf['source']['type'] == 'FTP':
44 ftp = FTP(conf['source']['ftp']['base_url'])
45 ftp.login()
46 ftp.cwd(conf['source']['ftp']['data_dir'])
47 l = ftp.nlst()
48 l.sort()
49 out = []
50 for s in l:
51 out.append({'code': s, 'label': s})
52 ftp.quit()
53 return out
54
55
57 """
58 List all the available years for a given MODIS product.
59 @param product_name: Code of MODIS product, e.g. 'MOD13Q1'
60 @return: An array of code/label objects.
61 """
62 if conf['source']['type'] == 'FTP':
63 ftp = FTP(conf['source']['ftp']['base_url'])
64 ftp.login()
65 ftp.cwd(conf['source']['ftp']['data_dir'])
66 ftp.cwd(product_name.upper())
67 l = ftp.nlst()
68 l.sort(reverse=True)
69 out = []
70 for s in l:
71 try:
72 float(s)
73 out.append({'code': s, 'label': s})
74 except ValueError:
75 pass
76 ftp.quit()
77 return out
78
79
81 """
82 List all the available days for a given MODIS product and year.
83 @param product_name: Code of MODIS product, e.g. 'MOD13Q1'
84 @param year: e.g. '2010'
85 @return: An array of code/label objects.
86 """
87 if conf['source']['type'] == 'FTP':
88 ftp = FTP(conf['source']['ftp']['base_url'])
89 ftp.login()
90 ftp.cwd(conf['source']['ftp']['data_dir'])
91 ftp.cwd(product_name.upper())
92 ftp.cwd(year)
93 l = ftp.nlst()
94 l.sort()
95 out = []
96 for s in l:
97 date = day_of_the_year_to_date(s, year).strftime('%d %B')
98 out.append({'code': s, 'label': date})
99 ftp.quit()
100 return out
101
102
104 """
105 List all the available layers for a given MODIS product, year and day.
106 @param product_name: Code of MODIS product, e.g. 'MOD13Q1'
107 @param year: e.g. '2010'
108 @param day: Day of the year, three digits, e.g. '017'
109 @return: An array of code/label/size objects.
110 """
111 ftp = FTP(conf['source']['ftp']['base_url'])
112 ftp.login()
113 ftp.cwd(conf['source']['ftp']['data_dir'])
114 ftp.cwd(product_name.upper())
115 ftp.cwd(year)
116 ftp.cwd(day)
117 ls = []
118 ftp.retrlines('MLSD', ls.append)
119 ftp.quit()
120 out = []
121 buffer = []
122 for line in ls:
123 try:
124 start = line.index('Size=')
125 end = line.index(';', start)
126 size = line[start + len('Size='):end]
127 start = line.index(product_name.upper())
128 file_name = line[start:]
129 if file_name not in buffer:
130 buffer.append(file_name)
131 file_path = 'ftp://' + conf['source']['ftp']['base_url'] + conf['source']['ftp']['data_dir']
132 file_path += product_name.upper() + '/' + year + '/' + day + '/'
133 file_path += line[start:]
134 h = file_name[2 + file_name.index('.h'):4 + file_name.index('.h')]
135 v = file_name[1 + file_name.index('v'):3 + file_name.index('v')]
136 label = 'H ' + h + ', V ' + v + ' (' + str(round((float(size) / 1000000), 2)) + ' MB)'
137 out.append({
138 'file_name': file_name,
139 'file_path': file_path,
140 'label': label,
141 'size': size
142 })
143 except:
144 pass
145 return out
146
147
149 """
150 List all the available layers for a given MODIS product, year and day.
151 @param product_name: Code of MODIS product, e.g. 'MOD13Q1'
152 @param year: e.g. '2010'
153 @param day: Day of the year, three digits, e.g. '017'
154 @param from_h: e.g. '05'
155 @param to_h: e.g. '05'
156 @param from_v: e.g. '05'
157 @param to_v: e.g. '05'
158 @return: An array of code/label/size objects.
159 """
160 if conf['source']['type'] == 'FTP':
161 ftp = FTP(conf['source']['ftp']['base_url'])
162 ftp.login()
163 ftp.cwd(conf['source']['ftp']['data_dir'])
164 ftp.cwd(product_name.upper())
165 ftp.cwd(year)
166 ftp.cwd(day)
167 ls = []
168 ftp.retrlines('MLSD', ls.append)
169 ftp.quit()
170 out = []
171 buffer = []
172 for line in ls:
173 try:
174 start = line.index('Size=')
175 end = line.index(';', start)
176 size = line[start + len('Size='):end]
177 start = line.index(product_name.upper())
178 file_name = line[start:]
179 if file_name not in buffer:
180 buffer.append(file_name)
181 if is_layer_in_the_range(file_name, from_h, to_h, from_v, to_v):
182 file_path = 'ftp://' + conf['source']['ftp']['base_url'] + conf['source']['ftp']['data_dir']
183 file_path += product_name.upper() + '/' + year + '/' + day + '/'
184 file_path += line[start:]
185 h = file_name[2 + file_name.index('.h'):4 + file_name.index('.h')]
186 v = file_name[1 + file_name.index('v'):3 + file_name.index('v')]
187 label = 'H ' + h + ', V ' + v + ' (' + str(round((float(size) / 1000000), 2)) + ' MB)'
188 out.append({
189 'file_name': file_name,
190 'file_path': file_path,
191 'label': label,
192 'size': size
193 })
194 except:
195 pass
196 return out
197
198
200 """
201 Check whether a given file is in the specified range, according to its name.
202 @param file_name: Name of the file.
203 @param from_h: Starting horizontal index of the range.
204 @param to_h: Ending horizontal index of the range.
205 @param from_v: Starting vertical index of the range.
206 @param to_v: Ending vertical index of the range.
207 @return: True if the file is in the range, false otherwise.
208 """
209 h_idx = len('.h') + file_name.index('.h')
210 h = int(file_name[h_idx:(2 + h_idx)])
211 v_idx = len(str(h) + 'v') + file_name.index(str(h) + 'v')
212 v = int(file_name[v_idx:(2 + v_idx)])
213 if int(from_h) <= h <= int(to_h) and int(from_v) <= v <= int(to_v):
214 return True
215 return False
216
217
220
221
223 """
224 List all the available layers for a given MODIS product, year and day.
225 @param product_name: Code of MODIS product, e.g. 'MOD13Q1'
226 @param year: e.g. '2010'
227 @param day: Day of the year, three digits, e.g. '017'
228 @param countries: GAUL codes, comma separated e.g. '18,25,34'
229 @type countries: String, comma separated
230 @return: An array of code/label/size objects.
231 """
232 countries_list = countries.split(',')
233 out = []
234 clean_out = []
235 file_names_buffer = []
236 for g2m in map:
237 if g2m['gaul_code'] in countries_list:
238 from_h = g2m['from_h']
239 to_h = g2m['to_h']
240 from_v = g2m['from_v']
241 to_v = g2m['to_v']
242 tmp = list_layers_subset(product_name, year, day, from_h, to_h, from_v, to_v)
243 out += tmp
244 for tmp in out:
245 if tmp['file_name'] not in file_names_buffer:
246 file_names_buffer.append(tmp['file_name'])
247 clean_out.append(tmp)
248 return clean_out
249
250
252 """
253 Convert a day of an year to a date
254 @param day: day of the year
255 @type string i.e. "020" or "20"
256 @param year: year of reference
257 @type string or int i.e. "2014" or 2014
258 @return: the date of the day/year i.e. "2012-01-20"
259 """
260 first_of_year = datetime.datetime(int(year), 1, 1).replace(month=1, day=1)
261 ordinal = first_of_year.toordinal() - 1 + int(day)
262 return datetime.date.fromordinal(ordinal)
263