Merge branch 'undefined' into 'master'

Upload New Script

See merge request !2

search_s5.py

+import os
+import logger
+import logging
+import requests
+import time
+import types
+import time
+from datetime import datetime, date, timedelta
+
+def search_s5():
+
+    # get important details for hub
+    sci_user = 's5pguest'
+	sci_passwd = 's5pguest'
+	sci_url = 'https://s5phub.copernicus.eu'
+
+    lastdate = "2018-01-01"
+	lastdate = datetime.strptime(datetime_str, '%y/%m/%d %H:%M:%S')
+
+    currentresults = 0
+	
+    # format last access time point
+    lastime = 'T' + str(lastdate.time()) + 'Z'
+    starting_date = str(lastdate.date()) + lastime
+
+
+    # ready to ask for products
+
+    retrieving_products = {}
+    products = {}
+
+    logging.info("Start searching %s from %s" %(sci_url,starting_date))
+    url = sci_url
+    url += '/dhus/search?q='
+    url += 'ingestiondate:[' + starting_date + ' TO NOW] '
+    url += 'platformname:Sentinel-5 '
+    url += '&format=json'
+
+    starting = -100
+    max_pagination_count = 100
+    currentresults = 0
+    while max_pagination_count == 100:
+
+        x = 0
+        while True:
+            # set starting point, increasing every time by 100
+            starting += max_pagination_count
+            url_pagination = url + '&start=%d&rows=100' % starting
+            session = requests.Session()
+            session.auth = (sci_user, sci_passwd)
+            session.stream = True
+            resp = session.get(url_pagination)
+            if resp.status_code == 200:
+                # update last access time point
+                h = hubs_lastaccess.objects.get(hubid_id=3, product='s3')
+                h.accessdate = datetime.now(tz=timezone.utc)
+                response = resp.json()
+
+                try:
+                    max_pagination_count = len(response['feed']['entry'])
+                except:
+                    x = 1
+
+                currentresults += max_pagination_count
+
+                if starting == 0:
+                    total_results = response['feed']['opensearch:totalResults']
+                    logging.info(total_results)
+                resp.close()
+                break
+
+            else:
+                logging.info('Something happened: \tStatus Code: %d, \tReason: %s,', resp.status_code,
+                             resp.reason)
+                logging.info('URL: %s' % url)
+                time.sleep(300)
+                x = 1
+                break
+
+        if x == 1:
+            break
+        print round(currentresults / float(total_results) * 100, 2), "%"
+
+        for prod in range(len(response['feed']['entry'])):
+            print response['feed']['entry'][prod]
+            exit()
+            try:
+                # get str metadata
+
+                meta_str = {meta['name']: meta['content'] for meta in response['feed']['entry'][prod]['str']}
+
+                #print response['feed']['entry'][prod]
+                #quicklook = response['feed']['entry'][prod]['link'][2]['href']
+
+                quicklook = ''
+                for meta in response['feed']['entry'][prod]['link']:
+                    if 'Quicklook' in meta['href']:
+                        quicklook = meta['href']
+                        break
+
+                identifier = meta_str['identifier']
+
+                for meta in response['feed']['entry'][prod]['int']:
+                    if 'relativeorbit' in meta['name']:
+                        ron = meta['content']
+                    elif 'orbitnum' in meta['name'] and not 'relative' in meta['name']:
+                        on = meta['content']
+
+                if on is None:
+                    on = 0
+                if ron is None:
+                    ron = 0
+
+
+                try:
+                    wkt_footprint = meta_str['footprint']
+                    # geom_footprint = GEOSGeometry(wkt_footprint, srid=4326)
+                    if not 'MULTI' in wkt_footprint:
+                        x = GEOSGeometry(wkt_footprint, srid=4326)
+                        geom_footprint = MultiPolygon(x, srid=4326)
+                    else:
+                        geom_footprint = GEOSGeometry(wkt_footprint, srid=4326)
+                except Exception as e:
+                    wkt_footprint = None
+                    geom_footprint = None
+
+
+                #tile = get_s2_new_naming_convention(identifier)[-2]
+                meta_date = {meta['name']: meta['content'] for meta in response['feed']['entry'][prod]['date']}
+                selected_metadata = {
+                    'uuid': meta_str['uuid'],
+                    'identifier': identifier,
+                    'filename': meta_str['filename'],
+                    'instrument': meta_str['instrumentshortname'],
+                    'product_type': meta_str['producttype'],
+                    'product_class':meta_str['productlevel'],
+                    'sensing_date': meta_date['beginposition'],
+                    'ingestion_date': meta_date['ingestiondate'],
+                    'wkt_footprint': wkt_footprint,
+                    'orbit_number': on, #response['feed']['entry'][prod]['int'][0]['content'],
+                    'relative_orbit_number': ron, #response['feed']['entry'][prod]['int'][1]['content'],
+                    'pass_direction': meta_str['orbitdirection'],
+                    'size': meta_str['size'],
+                    'url_download': response['feed']['entry'][prod]['link'][0]['href'],
+                    'url_checksum': response['feed']['entry'][prod]['link'][0]['href'][
+                                    :-6] + 'Checksum/Value/$value',
+                    'sensor_operational_mode': meta_str['sensoroperationalmode'],
+                    'timeliness': meta_str['timeliness'],
+                    'procfacilityname': meta_str['procfacilityname'],
+                    'procfacilityorg': meta_str['procfacilityorg'],
+                    'quicklook':quicklook,
+                    'geom_footprint':geom_footprint
+                }
+                k = (identifier)
+                products[k] = selected_metadata
+
+            except Exception as e:
+                #exc_type, exc_obj, exc_tb = sys.exc_info()
+                logging.info(e)
+                continue
+   
+    return products