Add tests for sharding, update relevant .gitignore

Requires the Python module 'sh'

sudo pip install sh

.gitignore

@@ -18,3 +18,4 @@ parser.h
 install_manifest.txt
 _CPack_Packages/*
 *.deb
+!.gitignore

test/.gitignore

@@ -0,0 +1,5 @@
+*.*-t*
+*-t*
+tempfile
+outfile
+!.gitignore

test/configs/blacklist_shard.conf

@@ -0,0 +1,25 @@
+# From IANA IPv4 Special-Purpose Address Registry
+# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+# Updated 2013-05-22
+
+0.0.0.0/8           # RFC1122: "This host on this network"
+10.0.0.0/8          # RFC1918: Private-Use
+100.64.0.0/10       # RFC6598: Shared Address Space
+127.0.0.0/8         # RFC1122: Loopback
+169.254.0.0/16      # RFC3927: Link Local
+172.16.0.0/12       # RFC1918: Private-Use
+192.0.0.0/24        # RFC6890: IETF Protocol Assignments
+192.0.2.0/24        # RFC5737: Documentation (TEST-NET-1)
+192.88.99.0/24      # RFC3068: 6to4 Relay Anycast
+#192.168.0.0/16      # RFC1918: Private-Use
+198.18.0.0/15       # RFC2544: Benchmarking
+198.51.100.0/24     # RFC5737: Documentation (TEST-NET-2)
+203.0.113.0/24      # RFC5737: Documentation (TEST-NET-3)
+240.0.0.0/4         # RFC1112: Reserved
+255.255.255.255/32  # RFC0919: Limited Broadcast
+
+# From IANA Multicast Address Space Registry
+# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml
+# Updated 2013-06-25
+
+224.0.0.0/4         # RFC5771: Multicast/Reserved

test/test_sharding.py

@@ -0,0 +1,72 @@
+import sh
+import unittest
+
+from sh import cut, grep, cat, wc, uniq, mv
+
+zmap_std_args = [ "-b",
+                  "configs/blacklist_shard.conf",
+                  "--seed=1234",
+                  "192.168.1.0/24",
+                  "--dryrun",
+                ]
+
+zmap = sh.Command("../src/zmap").bake(*zmap_std_args)
+
+def shard_file_name(shards, threads):
+    # Use naming convertion <shards>-t<threads>
+    return ''.join([str(shards), '-t', str(threads)])
+
+def output_file_name(shards, shard, threads):
+    # Use naming convention: <shards>.<shard>-t<threads>
+    return ''.join([str(shards), '.', str(shard), '-t', str(threads)])
+
+def parse(filename, **kwargs):
+    # cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | cut -d '.' -f 4 | sort -n | wc -l
+    return sh.sort(cut(cut(cut(grep(cat(filename), "ip"), d="|", f=2), d=" ", f=3), d=".", f=4), "-n", _out=kwargs.get("_out"))
+
+class TestSharding(unittest.TestCase):
+
+    NUM_IPS = 256
+
+    def setUp(self):
+        pass
+
+    def takeDown(self):
+        pass
+
+    def _runTest(self, shards, max_threads):
+        for threads in range(1, max_threads + 1):
+            for shard in range(0, shards):
+                with sh.sudo:
+                    outfile = output_file_name(shards, shard, threads)
+                    zmap(p=80, T=threads, shards=shards, shard=shard, _out="tempfile")
+                    parse("tempfile", _out=outfile)
+                    dup_lines = int(wc(uniq(cat(outfile), "-d"), "-l"))
+                    self.assertEqual(dup_lines, 0)
+                    shard_file = shard_file_name(shards, threads)
+                    if shard == 0:
+                        cat(outfile, _out=shard_file)
+                    else:
+                        cat(shard_file, outfile, _out="tempfile")
+                        mv("tempfile", shard_file)
+
+        for threads in range(1, max_threads + 1):
+            shard_file = shard_file_name(shards, threads)
+            num_lines = int(wc(cat(shard_file), "-l"))
+            self.assertEqual(num_lines, TestSharding.NUM_IPS)
+            dup_lines = int(wc(uniq(sh.sort(cat(shard_file), "-n"), "-d"), "-l"))
+            self.assertEqual(dup_lines, 0)
+
+    def testOneShard(self):
+        # Test with one shard
+        self._runTest(1, 4)
+
+
+    def testTwoShards(self):
+        self._runTest(2, 4)
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+