Added dmr1 for Slovenia and fixed srtm

.gitignore

@@ -18,6 +18,7 @@ gmted/
 etopo1/
 ned/
 ned_topobathy/
+dmr1/
 
 # tile generation directories
 tiles/

config.example.yaml

@@ -29,8 +29,11 @@ sources:
     tries: 100
   - type: greatlakes
   - type: srtm
-    url: http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/
-    mask-url: http://e4ftl01.cr.usgs.gov/SRTM/SRTMSWBD.003/2000.02.11/
+    url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11
+    mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11
+    username: 'earthdata username'
+    password: 'earthdata password'
+    auth-url: https://urs.earthdata.nasa.gov
   - type: ned13
     ftp_server: rockyftp.cr.usgs.gov
     base_path: vdelivery/Datasets/Staged/NED/13/IMG
@@ -40,5 +43,8 @@ sources:
   - type: ned_topobathy
     ftp_server: rockyftp.cr.usgs.gov
     base_path: vdelivery/Datasets/Staged/NED/19/IMG
+  - type: dmr1
+    uri: http://gis.arso.gov.si/lidar/dmr1
+    fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml
 logging:
   config: logging.example.config

config.slovenia.yaml

@@ -0,0 +1,49 @@
+---
+regions:
+  # san-francisco-bay_california:
+  #   bbox:
+  #     top: 38.719
+  #     left: -123.640
+  #     bottom: 36.791
+  #     right: -121.025
+  #   zoom_range: [0, 16]
+  slovenia:
+    bbox:
+      top: 46.88333333
+      left: 13.39027778
+      bottom: 45.40750000
+      right: 16.62694444 
+    zoom_range: [10, 16]
+outputs:
+  - type: skadi
+  - type: tiff
+  - type: terrarium
+  - type: normal
+sources:
+  - type: etopo1
+    url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip
+  - type: srtm
+    url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11
+    mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11
+    username: 'earthdata username'
+    password: 'earthdata password'
+    auth-url: https://urs.earthdata.nasa.gov
+  - type: dmr1
+    uri: http://gis.arso.gov.si/lidar/dmr1
+    fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml
+    bbox:
+      top: 46.88333333
+      left: 13.39027778
+      bottom: 45.40750000
+      right: 16.62694444 
+logging:
+  config: logging.example.config
+cluster:
+  queue:
+    type: fake
+store:
+  type: file
+  base_dir: 'render'
+source_store:
+  type: file
+  base_dir: 'source'

config.slovenia_test.yaml

@@ -0,0 +1,49 @@
+---
+regions:
+  # san-francisco-bay_california:
+  #   bbox:
+  #     top: 38.719
+  #     left: -123.640
+  #     bottom: 36.791
+  #     right: -121.025
+  #   zoom_range: [0, 16]
+  slovenia-test:
+    bbox:
+      top: 46.239346
+      left: 15.237007
+      bottom: 46.205735
+      right: 15.288334
+    zoom_range: [10, 16]
+outputs:
+  - type: skadi
+  - type: tiff
+  - type: terrarium
+  - type: normal
+sources:
+  - type: etopo1
+    url: https://www.ngdc.noaa.gov/mgg/global/relief/ETOPO1/data/bedrock/grid_registered/georeferenced_tiff/ETOPO1_Bed_g_geotiff.zip
+  - type: srtm
+    url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11
+    mask-url: http://e4ftl01.cr.usgs.gov/MEASURES/SRTMSWBD.003/2000.02.11
+    username: 'earthdata username'
+    password: 'earthdata password'
+    auth-url: https://urs.earthdata.nasa.gov
+  - type: dmr1
+    uri: http://gis.arso.gov.si/lidar/dmr1
+    fishnet_url: https://raw.githubusercontent.com/DavixDevelop/TerraLidar/master/mapzen_data/whitelist.yaml
+    bbox:
+      top: 46.239346
+      left: 15.237007
+      bottom: 46.205735
+      right: 15.288334
+logging:
+  config: logging.example.config
+cluster:
+  queue:
+    type: fake
+store:
+  type: file
+  base_dir: 'render'
+source_store:
+  type: file
+  base_dir: 'source'

docs/attribution.md

@@ -200,3 +200,18 @@ Attribution statement:
 Attribution statement:
 
 > © Commonwealth of Australia (Geoscience Australia) 2017.
+
+### Digital Elevation Model (DEM) of Slovenia derived from LiDAR 1 Meter Grid (DMR1)
+
+[Released](http://www.evode.gov.si/index.php?id=69) by Slovenia's [GURS](https://www.e-prostor.gov.si/), licensed under [Creative Commons 4.0](https://creativecommons.org/licenses/by/4.0/deed.sl)
+
+> Use Limitations:
+> General conditions for the use of geodetic data: https://www.e-prostor.gov.si/dostop-do-podatkov/dostop-do-podatkov/#tab1-1029
+>
+> Public access Restrictions:
+> No restrictions
+
+Attribution statement:
+
+> Geodetska uprava Republike Slovenije, LiDAR, DMR1 2015
+> © 2017 MOP - Geodetska uprava Republike Slovenije - Vse pravice pridržane.

docs/data-sources.md

@@ -21,6 +21,7 @@ The underlying data sources are a mix of:
 - [Kartverket](http://data.kartverket.no/download/content/digital-terrengmodell-10-m-utm-33)'s Digital Terrain Model, 10 meters over Norway
 - [LINZ](https://data.linz.govt.nz/layer/1768-nz-8m-digital-elevation-model-2012/), 8 meters over New Zealand
 - [SRTM](https://lta.cr.usgs.gov/SRTM) globally except high latitudes, 30 meters (90 meters nominal quality) in land areas
+- [DMR1](http://gis.arso.gov.si/evode/profile.aspx?id=atlas_voda_Lidar%40Arso&initialExtent=499500.5%2C109841.5%2C264.58333) derived from pre-processed Lidar, which contains only the terrain at 1m resolution, 1200m-1400m over Slovenia
 
 ### Footprints database
 
@@ -89,12 +90,12 @@ zoom   | ocean    | land
 **7**  | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60°
 **8**  | `ETOPO1` | `SRTM`, `NRCAN` in Canada, with `GMTED` in high latitudes above 60°
 **9**  | `ETOPO1` | `SRTM`, `NRCAN` in Canada, `EUDEM` in Europe, with `GMTED` in high latitudes above 60°
-**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
-**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
-**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
-**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
-**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
-**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway
+**10** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
+**11** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
+**12** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
+**13** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
+**14** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
+**15** | `ETOPO1`, `NED Topobathy` in California | `SRTM`, `data.gov.at` in Austria, `NRCAN` in Canada, `SRTM`, `NED/3DEP` 1/3 arcsec and 1/9 arcsec, `data.gov.uk` in United Kingdom, `INEGI` in Mexico, `ArcticDEM` in latitudes above 60°, `LINZ` in New Zealand, `Kartverket` in Norway, `DMR1` in Slovenia
 
 ## Sources native resolution
 

joerd/download.py

@@ -62,6 +62,15 @@ def get(url, options={}):
         # to restart from the beginning every time).
         accept_range = False
 
+        #auth username
+        username = options.get('username')
+
+        #auth pass
+        password = options.get('password')
+
+        #auth top level url
+        auth_url = options.get('auth-url')
+
         # we need to download _something_ if the file position is less than the
         # known size, or the size is unknown.
         while filesize is None or filepos < filesize:
@@ -77,6 +86,17 @@ def get(url, options={}):
 
             req = urllib2.Request(url)
 
+            # if the user provided a username and password, add Authorization
+            if username is not None:
+                redirectHandler = urllib2.HTTPRedirectHandler()
+                cookieProcessor = urllib2.HTTPCookieProcessor()
+                passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                passwordManager.add_password(None, auth_url, username, password)
+                authHandler = urllib2.HTTPBasicAuthHandler(passwordManager)
+                opener = urllib2.build_opener(redirectHandler,cookieProcessor,authHandler)
+                urllib2.install_opener(opener)
+                #req.add_header("Authorization", "Basic %s" % base64string) 
+
             # if the server supports accept range, and we have a partial
             # download then attemp to resume it.
             if accept_range and filepos > 0:
@@ -93,7 +113,9 @@ def get(url, options={}):
                 tmp.seek(0, os.SEEK_SET)
                 tmp.truncate(0)
 
+
             try:
+
                 f = urllib2.urlopen(req, timeout=timeout)
 
                 # try to get the filesize, if the server reports it.
@@ -183,4 +205,13 @@ def options(in_opts={}):
     tries = in_opts.get('tries', 10)
     out_opts['tries'] = int(tries)
 
-    return out_opts
+    username = in_opts.get('username', None)
+    out_opts['username'] = username
+
+    password = in_opts.get('password', None)
+    out_opts['password'] = password
+
+    auth_url = in_opts.get('auth-url', None)
+    out_opts['auth-url'] = auth_url
+
+    return out_opts

joerd/index.py

@@ -1,9 +1,11 @@
+from __future__ import print_function
 import pyqtree
 import yaml
 import logging
 import thread
 
 
+
 # Create an index given a YAML file consisting of a list of strings and a
 # function to parse it. Extra, fixed arguments for the function can be also
 # be given. Each object returned from the `parse_fn` should have a member
@@ -20,11 +22,12 @@ def create(index_file, bbox, parse_fn, *parse_args):
             if t:
                 idx.insert(bbox=t.bbox.bounds, item=t)
                 n += 1
+                print("\r Loaded... %d objects" % n, end="")
 
     logger.info("Created index with %d objects." % n)
     return idx
 
 
 # Returns a list of all objects intersecting the given bbox in the index.
 def intersections(idx, bbox):
-    return idx.intersect(bbox.bounds)
+    return idx.intersect(bbox.bounds)