Data serving scripts

From World Wind Wiki

(Difference between revisions)
Jump to: navigation, search
Revision as of 04:26, 6 July 2009 (edit)
Ammianus (Talk | contribs)
(Added java servlet data serving script)
← Previous diff
Revision as of 19:19, 22 October 2009 (edit) (undo)
164.67.235.138 (Talk)
(Python)
Next diff →
Line 348: Line 348:
from mod_python import apache from mod_python import apache
-import string 
import errno import errno
Line 385: Line 384:
# check parameter # check parameter
try: try:
- levelNum = string.atoi(level)+ levelNum = int(level)
- colNum = string.atoi(column)+ colNum = int(column)
- rowNum = string.atoi(row)+ rowNum = int(row)
except ValueError: except ValueError:
request_error(req) request_error(req)

Revision as of 19:19, 22 October 2009

Contents

PHP scripts

Images

<?
// This scipt is for serving out your own WW Cached tiles to World Wind, useful for own imagery and for working "off line"
// This script can be named what ever you want, it is controlled via a World Wind XML
//
// thanks to MaurizoZA and Nowak for the script

$X = $_GET['X'];
$Y = $_GET['Y'];
$L = $_GET['L'];
$T = $_GET['T'];

function addzeros($string){
       //echo $string;
       if(strlen($string) >= 4){
               return $string;
       }

       $string = "0" . "$string";

       if(strlen($string) < 4){
               $string = addzeros($string);
       }
       return $string;
}


$ext = ".jpg";

// Change the following to the location of your local root cache folder
$url = 'D:/cache/';
$doneurl = $url . $T . "/" . $L . "/" . addzeros($Y) . "/" . addzeros($Y) .
"_" . addzeros($X) . $ext;

// Debug tools
//header("Location: $doneurl");
//exit;
// print ($doneurl);
//exit;

     $tileData = file_exists($doneurl) ? file_get_contents($doneurl) :
false;
     if ($tileData === false) die();

     @header('Content-type: image/jpeg');
     print($tileData);

?>

Elevation

  • simple serving script:
<?
// This scipt is for serving out your own WW Cached tiles to World Wind, useful for own imagery and for working "off line"
// This script can be named what ever you want, it is controlled via a World Wind XML
//
// thanks to MaurizoZA and Nowak for the script

$X = $_GET['X'];
$Y = $_GET['Y'];
$L = $_GET['L'];
$T = $_GET['T'];

function addzeros($string){
       //echo $string;
       if(strlen($string) >= 4){
               return $string;
       }

       $string = "0" . "$string";

       if(strlen($string) < 4){
               $string = addzeros($string);
       }
       return $string;
}


$ext = ".bil.zip";

// Change the following to the location of your local root cache folder
$url = 'D:/cache/';
$doneurl = $url . $T . "/" . $L . "/" . addzeros($Y) . "/" . addzeros($Y) . "_" . addzeros($X) . $ext;

// Debug tools
//header("Location: $doneurl");
//exit;
// print ($doneurl);
//exit;

     $tileData = file_exists($doneurl) ? file_get_contents($doneurl) :
false;
     if ($tileData === false) die();

     @header('Content-type: application/zip');
     print($tileData);

?>
  • another script which can also create new tiles from tif files:
<?
//PHP script for dishing out BILs behind a web server or a geo-trawler.
//It will take a source TIF, use GDAL to cut up the source elevation data into WW tiles
//(and corresponding directory structure), and then GZip the files for delivery.

$szMapCacheDir="G:/CFSImagery/wwcache/DEM";
/* create the main cache directory if necessary */
if (!@is_dir($szMapCacheDir))
    makeDirs($szMapCacheDir);

/* get the various request parameters 
 * also need to make sure inputs are clean, especially those used to
 * build paths and filenames
 */
$X = isset( $_REQUEST['X'] ) ? intval($_REQUEST['X']) : 0;
$Y = isset( $_REQUEST['Y'] ) ? intval($_REQUEST['Y']) : 0;
$L = isset( $_REQUEST['L'] ) ? intval($_REQUEST['L']) : 0;
$T = isset( $_REQUEST['T'] ) ? intval($_REQUEST['T']) : 103;
$szExt = ".7z";

$szLevelcache = $szMapCacheDir."/$L";
$szYcache = sprintf($szLevelcache."/%04d",$Y);
if (!@is_dir($szYcache))
    makeDirs($szYcache);
$szCacheFile = sprintf($szYcache."/%04d_%04d".$szExt,$Y,$X);
$szIntFile = sprintf($szYcache."/%04d_%04d.bil",$Y,$X);;
/* Hit Test the cache tile */
if (!file_exists($szCacheFile) || $bForce){
 $lzts = 1.0;
 //Our Layer Zero Tile Size
 
 $lat1 = ($Y*$lzts*pow(0.5,$L))-90;
 $lon1 = ($X*$lzts*pow(0.5,$L))-180;
 //Lat2 and Lon2 as figured from tile size and level
 $lat2 = $lat1 + $lzts*pow(0.5,$L);
 $lon2 = $lon1 + $lzts*pow(0.5,$L);
 if($T==103){
 if(($lat1>-33)||($lon1<138)||($lat2<-36)||($lon2>140)){
  header("HTTP/1.0 404 Not Found");
  exit();
 }
 else{
  $gdalwarp = "gdalwarp.exe -te $lon1 $lat1 $lon2 $lat2 -ot Int16 -ts 150 150 -of ENVI ".
  "G:/CFSImagery/latlong/dem/hillsdem.tif ".$szIntFile;
  exec($gdalwarp);
  $za7 = "7za.exe a ".$szCacheFile." ".$szIntFile;
  exec($za7);
 }
 }
 if($T==104){
 if(($lat1>28)||($lon1<9)||($lat2<27)||($lon2>11)){
  header("HTTP/1.0 404 Not Found");
  exit();
 }
 else{
  $gdalwarp = "gdalwarp.exe -te $lon1 $lat1 $lon2 $lat2 -ot Int16 -ts 250 250 -of ENVI ".
  "G:/CFSImagery/latlong/dem/LibyaDuneslatlonint ".$szIntFile;
  exec($gdalwarp);
  $za7 = "7za.exe a ".$szCacheFile." ".$szIntFile;
  exec($za7);
 }
 }
}
$h = fopen($szCacheFile, "r");
header("Content-Type: "."application/x-7z-compressed");
header("Content-Length: " . filesize($szCacheFile));
header("Expires: " . date( "D, d M Y H:i:s GMT", time() + 31536000 ));
header("Cache-Control: max-age=31536000, must-revalidate" );
fpassthru($h);
fclose($h);
?>

Nowak's one-liner

<? print(file_get_contents('/path/'. sprintf('%u/%04u/%04u_%04u.jpg', $_GET['L'], $_GET['Y'], $_GET['Y'], $_GET['X']))); ?>

Perl script used for NLT Landsat and SRTM tile serving

#!/usr/bin/perl
use strict;
use CGI;
use Apache::Connection ();

my $r = shift;
my $c = $r->connection;
my $q = new CGI;

# Initialize
my $inDataset = "";
my $inLevel = "";
my $inX = "";
my $inY = "";
my $outInvalid = 0;
my $outDataType = 0;
my $outPathname = "";
my $outMimeType = "";
my $outExtension = "";
my $buff = "";
my $bytesRead = 0;
my $haveFile = 0;

# Get posted variables
if (length($q->param('T'))) {     # dataset
  $inDataset = $q->param('T');
  $inDataset = int($inDataset);
} else {
  $outInvalid = 1;
}

if (length($q->param('L'))) {    # level
  $inLevel = $q->param('L');
  $inLevel = int($inLevel);
} else {
  $outInvalid = 1;
}

if (length($q->param('X'))) {    # x
  $inX = $q->param('X');
  $inX = int($inX);
} else {
  $outInvalid = 1;
}

if (length($q->param('Y'))) {    # y
  $inY = $q->param('Y');
  $inY = int($inY);
} else {
  $outInvalid = 1;
}

# Verify input, create pathname
$outPathname = "/wwdata/";
$outMimeType = "";

if ($inDataset == 100) {
    $outPathname .= "100/";
    $outMimeType = "application/x-7z-compressed";
    $outExtension = ".7z";
    $outDataType = 1;
} elsif ($inDataset == 106) {
    $outPathname .= "106/";
    $outMimeType = "application/x-7z-compressed";
    $outExtension = ".7z";
    $outDataType = 1;
} elsif ($inDataset == 105) {
    $outPathname .= "105/";
    $outMimeType = "image/jpeg";
    $outExtension = ".jpg";
    $outDataType = 2;
} else {
    $outInvalid = 1;
}

if (($inLevel >= 0) && ($inLevel <= 9)) {
  $outPathname .= "${inLevel}/";
} else {
  $outInvalid = 1;
}

#0005_0006.7z
#
#pad with 3 zeroes, if less than 1000
if ($inY < 1000) {
    $inY = sprintf("%04u", $inY);
}
$outPathname .= "${inY}/${inY}_";

#pad with 3 zeroes, if less than 1000
if ($inX < 1000) {
    $inX = sprintf("%04u", $inX);
}
$outPathname .= "${inX}";
$outPathname .= "${outExtension}";


# Display (for debugging purposes)
#
#if (!$outInvalid) {
#  $r->content_type('text/plain');
#
#  print "$inDataset $inLevel $inX $inY\n";
#  print "$outPathname\n\n";
#}

if ($outInvalid) {
  $r->status(400);     # BAD_REQUEST
  exit 0;
}


# Do we have the file?
if (-e $outPathname && -r $outPathname) {
  $haveFile = 1;
} else {
  # handle error


#  if ($outDataType == 2) {
#     # Can't find the Landsat7 tile, push /wwdata/105/blank.jpg
#     $outPathname = "/wwdata/105/blank.jpg";
#     if (!(-e $outPathname) || !(-r $outPathname)) {
#       print "Can't find blank Landsat7 tile";
#       exit 0;
#     }
#  } else {
#     $r->content_type('text/plain');
#
#     print "Can't find tile\n";
#     print "$inDataset $inLevel $inX $inY $outMimeType $outExtension";


     $r->status(404);      # NOT_FOUND
     exit 0;
#  }
}
 
# Send file

# Set up headers
$r->content_type("$outMimeType");
$r->set_content_length((-s($outPathname)));

# Open and display file
open my $fh, "$outPathname";
binmode $fh;
binmode STDOUT;
  
$bytesRead = read ($fh, $buff, 8192);
while ($bytesRead) {
  print $buff;
  $r->rflush;
  last if $c->aborted;
  $bytesRead = read ($fh, $buff, 8192);
}
close $fh;
exit 0;

Python

# simple python worldwind tile server, v 1.0 2006
# author: Frank Kuehnel (RIACS)
 
from mod_python import apache
import errno

#todo: make more robust + handle various datasets

datasetBasename = '/Users/admin/apollo/'

def request_error(req):
	req.content_type = 'text/plain'
	req.send_http_header()
	req.write('Invalid WorldWind tile request\n')

def handler(req):
	global datasetBasename	

	# we want to have parameters at least
	if req.args:
		args = req.args
	else:
		request_error(req)
		return apache.OK
		
	args = req.args
	for elem in args.split('&'):
		key   = elem[0:2]
		value = elem[2:]
		if key == 'T=':
			dataSet = value
		elif key == 'L=':
			level = value
		elif key == 'X=':
			column = value
		elif key == 'Y=':
			row = value

	# check parameter
	try:
		levelNum = int(level)
		colNum = int(column)
		rowNum = int(row)
	except ValueError:
		request_error(req)
		return apache.OK

	# load image file
	try:
		filepath = '%u/%04u/' % (levelNum, rowNum)
		filename = '%04u_%04u.jpg' % (rowNum, colNum)
		fullname = datasetBasename + filepath + filename
		input = open(fullname,'rb')
		data  = input.read()
		input.close()
	except IOError, err:
		if err.errno == errno.ENOENT:
                        req.content_type = 'text/plain'
			req.send_http_header()
                        req.write('image tile ' + fullname + ' not available')
                        return apache.OK

	# output image
	req.content_type = 'image/jpeg'
	req.send_http_header()
	req.write(data)

	return apache.OK

IIS

WorldWindMaps.aspx.cs (the script itself)

using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using System.IO;
using System.Web.Configuration;

public partial class WorldWindMaps : System.Web.UI.Page
{
    protected void Page_Load(object sender, EventArgs e)
    {

        int X = Convert.ToInt32(Request.QueryString["X"].ToString());
        int Y = Convert.ToInt32(Request.QueryString["Y"].ToString());
        int L = Convert.ToInt32(Request.QueryString["L"].ToString());
        string T = Request.QueryString["T"].ToString(); //T = Dataset in WorldWind parlance

        string filePath = WebConfigurationManager.AppSettings[T];

        string fileExt = WebConfigurationManager.AppSettings[T + "_EXT"];

        filePath += L.ToString() + "\\" + Y.ToString("0000") + "\\" + Y.ToString("0000") + "_" + X.ToString("0000") + fileExt;

        FileInfo file = new FileInfo(filePath);


        if (!file.Exists && T == "SRTM")
        {
            throw new HttpException(404, "Not Found");
            Response.End();
        }

        else if (!file.Exists)
        {
            
		string blankfilepath = WebConfigurationManager.AppSettings[T] + "0" + "\\" + "0000" + "\\" + "blank" + fileExt;
		FileInfo blankfile = new FileInfo(blankfilepath);
      	
		Response.ClearContent();
        	Response.AddHeader("Content-Length", blankfile.Length.ToString());
        	Response.ContentType = WebConfigurationManager.AppSettings[T + "_MIME"];
        	Response.TransmitFile(blankfile.FullName);
        }
	else
	{

        	Response.ClearContent();
        	Response.AddHeader("Content-Length", file.Length.ToString());
        	Response.ContentType = WebConfigurationManager.AppSettings[T + "_MIME"];
        	Response.TransmitFile(file.FullName);
	}
    }
}

sample Web.config file

<?xml version="1.0" encoding="utf-8" ?>
<configuration>

   <!--
     Note: WorldWind Tile Server
   -->
   
	<appSettings>
			
        <!-- WorldWind Image Services -->
        <add key="SRTM" value="C:\World Wind Cache\Cache\Earth\SRTM\"/>
        <add key="LANDSAT" value="C:\World Wind Cache\Cache\Earth\Images\NASA Landsat Imagery\NLT Landsat7 (Visible Color)\"/>

        <!-- WorldWind Image file extensions -->
        <add key="SRTM_EXT" value=".bil"/>
        <add key="LANDSAT_EXT" value=".jpg"/>

        <!-- WorldWind Image content (MIME) types -->
        <add key="SRTM_MIME" value="application/octet-stream"/>
        <add key="LANDSAT_MIME" value="image/jpeg"/>

	</appSettings>
     
  <system.web>

    <compilation 
         defaultLanguage="c#"
         debug="true"
    />

    <customErrors 
    mode="Off" 
    /> 

    <authentication mode="Windows" /> 

    <authorization>
        <allow users="*" /> <!-- Allow all users -->
            <!--  <allow     users="[comma separated list of users]"
                             roles="[comma separated list of roles]"/>
                  <deny      users="[comma separated list of users]"
                             roles="[comma separated list of roles]"/>
            -->
    </authorization>

    <trace
        enabled="false"
        requestLimit="10"
        pageOutput="false"
        traceMode="SortByTime"
		localOnly="true"
    />

    <sessionState 
            mode="InProc"
            stateConnectionString="tcpip=127.0.0.1:42424"
            sqlConnectionString="data source=127.0.0.1;Trusted_Connection=yes"
            cookieless="false" 
            timeout="20" 
    />

    <globalization 
            requestEncoding="utf-8" 
            responseEncoding="utf-8" 
   />
   
 </system.web>

</configuration>

Java Servlet

This was a script created for verifying and testing download code, it is simple to set up. Developed on JDK 1.6/Tomcat 6.0, Windows Vista x64 Home Premium

Tested this with the wwd-nltls7-g20-l0 (GeoCover 2000) cache pack installed locally. Using WorldWorld.NET 1.4

Servlet class TileRequestHandler.java

package org.wwc.ammianus.tomcat.tileserver;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.HashMap;

import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

/**
 * <p>This servlet handles requests from the World Wind.NET application and returns 
 * tile images or error codes as appropriate. This was a port to Java of a ASP.NET file posted
 * at <a>http://worldwindcentral.com/wiki/Data_serving_scripts</a></p>
 * 
 * @author ammianus
 * @version 0.1
 * 
 * 
 */
public class TileRequestHandler extends HttpServlet {
	
	/**
	 * Generated by Eclipse
	 */
	private static final long serialVersionUID = -6596187549520730077L;
	/**
	 * settings Map which theoretically could be extended for any layer used in WW
	 * for each layer, use the <DataSetName></DataSetName> tag value from @Images.xml file
	 * create three map entries <DataSetName>, <DataSetName>_EXT, <DataSetName>_MIME
	 */
	private HashMap<String, String> _settings;
	
	private int _requestNumber;
	
    /**
     * @see HttpServlet#HttpServlet()
     */
    public TileRequestHandler() {
        super();
        //use these setting which theoretically could be extended for any layer used in WW
        //for each layer, use the <DataSetName></DataSetName> tag value from @Images.xml file
        //create three map entries <DataSetName>, <DataSetName>_EXT, <DataSetName>_MIME
        _settings = new HashMap<String,String>();
        _settings.put("geocover2000", "C:\\Users\\Public\\WorldWindCache\\Cache\\Earth\\Images\\NASA Landsat Imagery\\Geocover 2000");
        _settings.put("geocover2000_EXT", ".jpg");
        _settings.put("geocover2000_MIME", "image/jpeg");
        
        _requestNumber = 0;
    }

    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    	//increment counter
    	_requestNumber++;
    	
    	//Un-comment this to test whether WW handles 503 return code with Retry-After header
    	//
    	//first request we will send an error with a retry-after header
    	//
    	/*if(_requestNumber == 1){
    		response.setStatus(503);
    		//this appears to be case insensitive for WorldWind, I just picked a random retry-after period
    		response.setHeader("retry-after", "38");
    		System.out.println(_requestNumber+":Response: 503, with retry-after: 38");
    		return;
    	}*/
    	
    	try{
    	
    		
    	//column	
    	int X = Integer.parseInt(request.getParameter("X").toString());
    	//row
    	int Y = Integer.parseInt(request.getParameter("Y").toString());
    	//level
    	int L = Integer.parseInt(request.getParameter("L").toString());
    	//DataSet name
    	String T = request.getParameter("T").toString(); //T = Dataset in WorldWind parlance
    	
    	
    	//Un-comment this to test whether WW handles 500 return code
    	//
    	//if(T.equals("geocover2000")){
    	//	throw new Exception("Force return of 500 - Server Error");	
    	//}
    	
    	//Arguments are in this format URL sent from world wind
    	//e.g. http://localhost:8080/TomcatTileServer/TileRequestHandler?T=geocover2000&L=0&X=47&Y=21
    	String filePath = _settings.get(T);//WebConfigurationManager.AppSettings[T];

    	String fileExt = _settings.get(T + "_EXT");
    	if(L < 3){
    		DecimalFormat df = new DecimalFormat("0000");	
    		filePath += "\\"+L + "\\" + df.format(Y) + "\\" + df.format(Y) + "_" + df.format(X) + fileExt;
    	}else{
    		filePath += "\\level"+(L+1)+fileExt;
    	}
    	//request.get
    	System.out.println(_requestNumber+":Requested File: "+filePath);
    	File file = new File(filePath);

    	//if file (image for requested tile) is not found on server, return 404
    	if (!file.exists())
    	{
    		response.sendError(HttpServletResponse.SC_NOT_FOUND, "Not Found");
    	}
    	else
    	{
    		//set content length to file size
    		response.setContentLength((int) file.length());
    		//get mime type for this layer
    		response.setContentType(_settings.get(T + "_MIME"));
    		//get the output stream for the response, this will have the file written to it
    		ServletOutputStream stream = response.getOutputStream();
    		
    		//read the file contents and write them to the outputstream
    		FileInputStream fis = new FileInputStream(file);
    		byte[] bytes = new byte[1024];
    		int counter = 0;
    		while(counter < file.length()){
    			fis.read(bytes);
    			counter += 1024;
    			stream.write(bytes);
    		}
    		fis.close();
    		stream.flush();
    		stream.close();
    		
    		//done with response
    	}
    	}catch(Exception e){
    		//uncaught Exception return 500 error
    		response.sendError(500,e.getMessage());
    		System.out.println(_requestNumber+":Response: send 500 error");
    	}

    }

	/**
	 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
	 */
	protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		// TODO Auto-generated method stub
	}

}

Example WEB-INF\web.xml

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" id="WebApp_ID" version="2.5">
  <display-name>TomcatTileServer</display-name>
  <welcome-file-list>
    <welcome-file>index.html</welcome-file>
    <welcome-file>index.htm</welcome-file>
    <welcome-file>index.jsp</welcome-file>
    <welcome-file>default.html</welcome-file>
    <welcome-file>default.htm</welcome-file>
    <welcome-file>default.jsp</welcome-file>
  </welcome-file-list>
  <servlet>
    <description></description>
    <display-name>TileRequestHandler</display-name>
    <servlet-name>TileRequestHandler</servlet-name>
    <servlet-class>org.wwc.ammianus.tomcat.tileserver.TileRequestHandler</servlet-class>
  </servlet>
  <servlet-mapping>
    <servlet-name>TileRequestHandler</servlet-name>
    <url-pattern>/TileRequestHandler</url-pattern>
  </servlet-mapping>
</web-app>

Note

Some of these scripts were designed for World Wind 1.3 which used 7-zip compression for .bil tiles. Version 1.4 supports both 7-zip and zip.

Personal tools