- Fixed "Z" type shapefiles to also read measure values as per the Es…

…ri specification

- Check for "nodata" values (< -10e38) in measures as per the Esri specification and return "None"
- Added the ability to read from Python file-like objects

git-svn-id: https://pyshp.googlecode.com/svn/trunk@55 6bd14dac-74e7-11de-ab85-01204adfcb19

Python2/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>pyshp</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

Python2/.pydevproject

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/pyshp</path>
+</pydev_pathproperty>
+</pydev_project>

Python2/setup.py

@@ -2,7 +2,7 @@
 import usage
 
 setup(name='pyshp',
-      version='1.0.6',
+      version='1.1.0',
       description='Pure Python read/write support for ESRI Shapefile format',
       long_description=usage.__doc__.strip(),
       author='Joel Lawhead',

Python2/shapefile.py

@@ -1,9 +1,9 @@
 """
 shapefile.py
 Provides read and write support for ESRI Shapefiles.
-author: jlawhead<at>nvs-inc.com
-date: 20110904
-version: 1.0.8
+author: jlawhead<at>geospatialpython.com
+date: 20110921
+version: 1.1.0
 """
 
 from struct import pack, unpack, calcsize, error
@@ -75,7 +75,7 @@ class Reader:
     efficiently as possible. Shapefiles are usually not large
     but they can be.
     """
-    def __init__(self, shapefile=None):
+    def __init__(self, *args, **kwargs):
         self.shp = None
         self.shx = None
         self.dbf = None
@@ -85,7 +85,30 @@ def __init__(self, shapefile=None):
         self.numRecords = None
         self.fields = []
         self.__dbfHdrLength = 0
-        self.load(shapefile)
+        # See if a shapefile name was passed as an argument
+        if len(args) > 0:
+            if type(args[0]) is type("stringTest"):
+                self.load(args[0])
+                return
+        if "shp" in kwargs.keys():
+            if hasattr(kwargs["shp"], "read"):
+                self.shp = kwargs["shp"]
+                if hasattr(self.shp, "seek"):
+                    self.shp.seek(0)
+            if "shx" in kwargs.keys():
+                if hasattr(kwargs["shx"], "read"):
+                    self.shx = kwargs["shx"]
+                    if hasattr(self.shx, "seek"):
+                        self.shx.seek(0)
+        if "dbf" in kwargs.keys():
+            if hasattr(kwargs["dbf"], "read"):
+                self.dbf = kwargs["dbf"]
+                if hasattr(self.dbf, "seek"):
+                    self.dbf.seek(0)
+        if self.shp or self.dbf:        
+            self.load()
+        else:
+            raise ShapefileException("Shapefile Reader requires a shapefile or file-like object.")
 
     def load(self, shapefile=None):
         """Opens a shapefile from a filename or file-like
@@ -97,22 +120,30 @@ def load(self, shapefile=None):
             self.shapeName = shapeName
             try:
                 self.shp = file("%s.shp" % shapeName, "rb")
-                self.__shpHeader()
-            except IOError: pass
+            except IOError:
+                raise ShapefileException("Unable to open %s.shp" % shapeName)
             try:
                 self.shx = file("%s.shx" % shapeName, "rb")
-            #       self.__shapeIndex()
-            except IOError: pass
+            except IOError:
+                raise ShapefileException("Unable to open %s.shx" % shapeName)
             try:
                 self.dbf = file("%s.dbf" % shapeName, "rb")
-                self.__dbfHeader()
-            except IOError: pass
+            except IOError:
+                raise ShapefileException("Unable to open %s.dbf" % shapeName)
+        if self.shp:
+            self.__shpHeader()
+        if self.dbf:
+            self.__dbfHeader()
 
     def __getFileObj(self, f):
         """Checks to see if the requested shapefile file object is
         available. If not a ShapefileException is raised."""
         if not f:
-            raise ShapefileException("Required file not available.")
+            raise ShapefileException("Shapefile Reader requires a shapefile or file-like object.")
+        if self.shp and self.shpLength is None:
+            self.load()
+        if self.dbf and len(self.fields) == 0:
+            self.load()
         return f
 
     def __restrictIndex(self, i):
@@ -127,19 +158,21 @@ def __restrictIndex(self, i):
 
     def __shpHeader(self):
         """Reads the header information from a .shp or .shx file."""
-        f = self.__getFileObj(self.shp)
+        if not self.shp:
+            raise ShapefileException("Shapefile Reader requires a shapefile or file-like object. (no shp file found")
+        shp = self.shp
         # File length (16-bit word * 2 = bytes)
-        f.seek(24)
-        self.shpLength = unpack(">i", f.read(4))[0] * 2
+        shp.seek(24)
+        self.shpLength = unpack(">i", shp.read(4))[0] * 2
         # Shape type
-        f.seek(32)
-        self.shapeType= unpack("<i", f.read(4))[0]
+        shp.seek(32)
+        self.shapeType= unpack("<i", shp.read(4))[0]
         # The shapefile's bounding box (lower left, upper right)
-        self.bbox = _Array('d', unpack("<4d", f.read(32)))
+        self.bbox = _Array('d', unpack("<4d", shp.read(32)))
         # Elevation
-        self.elevation = _Array('d', unpack("<2d", f.read(16)))
+        self.elevation = _Array('d', unpack("<2d", shp.read(16)))
         # Measure
-        self.measure = _Array('d', unpack("<2d", f.read(16)))
+        self.measure = _Array('d', unpack("<2d", shp.read(16)))
 
     def __shape(self):
         """Returns the header info and geometry for a single shape."""
@@ -175,9 +208,10 @@ def __shape(self):
             (zmin, zmax) = unpack("<2d", f.read(16))
             record.z = _Array('d', unpack("<%sd" % nPoints, f.read(nPoints * 8)))
         # Read m extremes and values
-        if shapeType in (23,25,28):
+        if shapeType in (13,15,18,23,25,28,31):
             (mmin, mmax) = unpack("<2d", f.read(16))
-            record.m = _Array('d', unpack("%sd" % nPoints, f.read(nPoints * 8)))
+            # Measure values less than -10e38 are nodata values according to the spec
+            record.m = [m if m > -10e38 else None for m in _Array('d', unpack("%sd" % nPoints, f.read(nPoints * 8)))]
         # Read a single point
         if shapeType in (1,11,21):
             record.points = [_Array('d', unpack("<2d", f.read(16)))]
@@ -192,59 +226,64 @@ def __shape(self):
     def __shapeIndex(self, i=None):
         """Returns the offset in a .shp file for a shape based on information
         in the .shx index file."""
-        f = self.__getFileObj(self.shx)
-        if not f:
+        shx = self.shx
+        if not shx:
             return None
         if not self._offsets:
             # File length (16-bit word * 2 = bytes) - header length
-            f.seek(24)
-            shxRecordLength = (unpack(">i", f.read(4))[0] * 2) - 100
+            shx.seek(24)
+            shxRecordLength = (unpack(">i", shx.read(4))[0] * 2) - 100
             numRecords = shxRecordLength / 8
             # Jump to the first record.
-            f.seek(100)
+            shx.seek(100)
             for r in range(numRecords):
                 # Offsets are 16-bit words just like the file length
-                self._offsets.append(unpack(">i", f.read(4))[0] * 2)
-                f.seek(f.tell() + 4)
+                self._offsets.append(unpack(">i", shx.read(4))[0] * 2)
+                shx.seek(shx.tell() + 4)
         if not i == None:
             return self._offsets[i]
 
     def shape(self, i=0):
         """Returns a shape object for a shape in the the geometry
         record file."""
-        f = self.shp
+        shp = self.__getFileObj(self.shp)
         i = self.__restrictIndex(i)
         offset = self.__shapeIndex(i)
         if not offset:
             # Shx index not available so use the full list.
             shapes = self.shapes()
             return shapes[i]
-        f.seek(offset)
+        shp.seek(offset)
         return self.__shape()
 
     def shapes(self):
         """Returns all shapes in a shapefile."""
-        self.shp.seek(100)
+        shp = self.__getFileObj(self.shp)
+        shp.seek(100)
         shapes = []
-        while self.shp.tell() < self.shpLength:
+        while shp.tell() < self.shpLength:
             shapes.append(self.__shape())
         return shapes
 
     def __dbfHeaderLength(self):
         """Retrieves the header length of a dbf file header."""
         if not self.__dbfHdrLength:
-            f = self.__getFileObj(self.dbf)
+            if not self.dbf:
+                raise ShapefileException("Shapefile Reader requires a shapefile or file-like object. (no dbf file found)")
+            dbf = self.dbf
             (self.numRecords, self.__dbfHdrLength) = \
-                    unpack("<xxxxLH22x", f.read(32))
+                    unpack("<xxxxLH22x", dbf.read(32))
         return self.__dbfHdrLength
 
     def __dbfHeader(self):
         """Reads a dbf header. Xbase-related code borrows heavily from ActiveState Python Cookbook Recipe 362715 by Raymond Hettinger"""
-        f = self.__getFileObj(self.dbf)
+        if not self.dbf:
+            raise ShapefileException("Shapefile Reader requires a shapefile or file-like object. (no dbf file found)")
+        dbf = self.dbf
         headerLength = self.__dbfHeaderLength()
         numFields = (headerLength - 33) // 32
         for field in range(numFields):
-            fieldDesc = list(unpack("<11sc4xBB14x", f.read(32)))
+            fieldDesc = list(unpack("<11sc4xBB14x", dbf.read(32)))
             name = 0
             idx = 0
             if "\x00" in fieldDesc[name]:
@@ -254,7 +293,7 @@ def __dbfHeader(self):
             fieldDesc[name] = fieldDesc[name][:idx]
             fieldDesc[name] = fieldDesc[name].lstrip()
             self.fields.append(fieldDesc)
-        terminator = f.read(1)
+        terminator = dbf.read(1)
         assert terminator == "\r"
         self.fields.insert(0, ('DeletionFlag', 'C', 1, 0))
 
@@ -757,17 +796,24 @@ def saveDbf(self, target):
         self.__dbfHeader()
         self.__dbfRecords()
 
-    def save(self, target=""):
+    def save(self, target=None, shp=None, shx=None, dbf=None):
         """Save the shapefile data to three files or
-        three file-like objects. SHP and DBF files can
+        three file-like objects. SHP and DBF files can also
         be written exclusively using saveShp, saveShx, and saveDbf respectively."""
         # TODO: Create a unique filename for target if None.
-        self.saveShp(target)
-        self.shp.close()
-        self.saveShx(target)
-        self.shx.close()
-        self.saveDbf(target)
-        self.dbf.close()
+        if shp:
+            self.saveShp(shp)
+        if shx:
+            self.saveShx(shx)
+        if dbf:
+            self.saveDbf(dbf)
+        elif target:
+            self.saveShp(target)
+            self.shp.close()
+            self.saveShx(target)
+            self.shx.close()
+            self.saveDbf(target)
+            self.dbf.close()
 
 class Editor(Writer):
     def __init__(self, shapefile=None, shapeType=POINT, autoBalance=1):

Python2/usage.py

@@ -3,32 +3,39 @@
 Python Shapefile Library
 ========================
 :Author: Joel Lawhead <[email protected]>
-:Revised: August 31, 2011
+:Revised: September 21, 2011
 
 Overview
 --------
 
-The Python Shapefile Library (PSL) provides read and write support for the ESRI
+The Python Shapefile Library (pyshp) provides read and write support for the ESRI
 Shapefile format. The Shapefile format is a popular Geographic Information
-System vector data format.
+System vector data format created by Esri.  For more information about this format 
+please read the well-written "ESRI Shapefile Technical Description - July 1998".  
+The Esri document describes the shp and shx file formats.  However a third file
+format called dbf is also required.  This format is documented on the web as the
+"XBase File Format Description" and is a simple file-based database format created
+in the 1960's.  Both the Esri and XBase file-formats are very simple in design and
+memory efficient which is part of the reason the shapefile format remains popular
+despite the numerous ways to store and exchange GIS data available today. 
 
 This documentation covers the Python 2.x-compatible version of the library.  A
 Python 3-compatible version is available in the Subversion trunk of the pyshp 
 project on Google Code.
 
-This document provides usage examples for using the Python Shapefile Library.  
+This document provides examples for using pyshp to read and write shapefiles.  
 
 Currently the sample census blockgroup shapefile referenced in the examples is
 only available on the google code project site at http://code.google.com/p/pyshp.
-These examples are straight-forward and you can also easily run these simple 
-examples on your own shapefiles manually. Other examples for specific topics are
-continually added to the pyshp wiki on google code and the blog:
+These examples are straight-forward and you can also easily run them against your 
+own shapefiles manually with minimal modification. Other examples for specific 
+topics are continually added to the pyshp wiki on google code and the blog
 GeospatialPython.com.
 
 Important: For information about map projections, shapefiles,
-and Python please visit:http://code.google.com/p/pyshp/wiki/MapProjections
+and Python please visit: http://code.google.com/p/pyshp/wiki/MapProjections
 
-I sincerely hope this libarary eliminates the mundane distraction of simply 
+I sincerely hope this library eliminates the mundane distraction of simply 
 reading and writing data, and allows you to focus on the challenging and FUN
 part of your project. 
 
@@ -64,7 +71,22 @@
 >>> sf = shapefile.Reader("shapefiles/blockgroups.dbf")
 
 
-OR any of the other 5+ formats which are potentially part of a shapefile.
+OR any of the other 5+ formats which are potentially part of a shapefile. 
+The library does not care
+
+You can ALSO load shapefiles from any Python file-like object using keyword
+arguments to specify any of the three files.  This feature is very powerful
+and allows you to load shapefiles from a url, from a zip file, serialized
+object, or in some cases a database.
+
+>>> myshp = file("shapefiles/blockgroups.shp", "rb")
+>>> mydbf = file("shapefiles/blockgroups.dbf", "rb")
+>>> r = shapefile.Reader(shp=myshp, dbf=mydbf)
+
+Notice in the examples above the shx file is never used.  The shx file is a 
+very simple fixed-record index for the variable length records in the shp file.
+This file is optional for reading.  If it's available pyshp will use the shx file
+to access shape records a little faster but will do just fine without it.
 
 Reading Geometry
 ................