parse by bytes (not newlines)

lib/slither/definition.rb

@@ -5,7 +5,7 @@ class Definition
     def initialize(options = {})
       @sections = []
       @templates = {}
-      @options = { :align => :right }.merge(options)
+      @options = { :align => :right, :by_bytes => false }.merge(options)
     end
 
     def section(name, options = {}, &block)

lib/slither/parser.rb

@@ -14,7 +14,10 @@ def parse()
       @file.each_line do |line|
         line.chomp! if line
         @definition.sections.each do |section|
-          parsed = fill_content(line, section, parsed)
+          if section.match(line)
+            validate_length(line, section)
+            parsed = fill_content(line, section, parsed)
+          end
         end
       end
 
@@ -24,20 +27,45 @@ def parse()
       parsed
     end
 
+    def parse_by_bytes
+      parsed = {}
+
+      byte_length = @definition.sections.map{|sec| sec.length }.max  # all sections must be the same length for parse_by_bytes
+      while record = @file.read(byte_length)
+        record.force_encoding @file.external_encoding
+
+        raise(Slither::LineWrongSizeError, "newline character was not at the end of byte group") unless newline?(record[-1])
+        #record.chomp!
+        @definition.sections.each do |section|
+          if section.match(record)
+            parsed = fill_content(record, section, parsed)
+          end
+        end
+      end
+
+      @definition.sections.each do |section|
+        raise(Slither::RequiredSectionNotFoundError, "Required section '#{section.name}' was not found.") unless parsed[section.name] || section.optional
+      end
+      parsed
+    end
+
     private
 
       def fill_content(line, section, parsed)
-        if section.match(line)
-          validate_length(line, section)
-          parsed[section.name] = [] unless parsed[section.name]
-          parsed[section.name] << section.parse(line)
-        end
+        parsed[section.name] = [] unless parsed[section.name]
+        parsed[section.name] << section.parse(line)
         parsed
       end
 
       def validate_length(line, section)
-        raise Slither::LineTooLongError, "Line too long (#{line.length} when it should be #{section.length})" if line.length > section.length
-        raise Slither::LineTooShortError, "Line too short (#{line.length} when it should be #{section.length})" if line.length < section.length
+        raise Slither::LineWrongSizeError, "Line wrong size: (#{line.length} when it should be #{section.length})" if line.length != section.length
+      end
+
+      def newline?(char)
+        # \n or LF -> 10
+        # \r or CR -> 13
+        return false unless char && !char.empty?
+        [10, 13].any?{|code| char.ord == code}
       end
 
   end

lib/slither/slither.rb

@@ -7,8 +7,7 @@ class RequiredSectionNotFoundError < StandardError; end
   class RequiredSectionEmptyError < StandardError; end
   class FormattedStringExceedsLengthError < StandardError; end
   class ColumnMismatchError < StandardError; end
-  class LineTooLongError < StandardError; end
-  class LineTooShortError < StandardError; end
+  class LineWrongSizeError < StandardError; end
 
 
   def self.define(name, options = {}, &block)
@@ -42,7 +41,7 @@ def self.parseIo(io, definition_name)
     definition = definition(definition_name)
     raise ArgumentError, "Definition name '#{definition_name}' was not found." unless definition
     parser = Parser.new(definition, io)
-    parser.parse
+    definition.options[:by_bytes] ? parser.parse_by_bytes : parser.parse
   end
 
   private

spec/definition_spec.rb

@@ -16,11 +16,12 @@
     end
 
     it "should override the default if :align is passed to the section" do
-      section = mock('section').as_null_object
-      Slither::Section.should_receive(:new).with('name', {:align => :left}).and_return(section)
       d = Slither::Definition.new
       d.options[:align].should == :right
       d.section('name', :align => :left) {}
+      section = nil
+      d.sections.each { |sec| section = sec if sec.name == 'name' }
+      section.options[:align].should eq(:left)
     end
   end
 

spec/parser_spec.rb

@@ -12,7 +12,6 @@
         end
         d.body do |b|
           b.trap { |line| line[0,4] != 'HEAD' &&  line[0,4] != 'FOOT'}
-          #b.trap { |line| line[0,4] == '    ' }
           b.column :first, 10
           b.column :last, 10
         end
@@ -60,17 +59,68 @@
       @definition.sections[0].optional = true
       @definition.sections[2].optional = true
       @file_io.stub(:each_line).and_yield('abc'*20).and_yield(nil)
-      lambda { @parser.parse }.should raise_error(Slither::LineTooLongError)
+      lambda { @parser.parse }.should raise_error(Slither::LineWrongSizeError)
     end
 
     it "should raise an error if the line is too short" do
       @definition.sections[0].optional = true
       @definition.sections[2].optional = true
       @file_io.stub(:each_line).and_yield('abc').and_yield(nil)
-      lambda { @parser.parse }.should raise_error(Slither::LineTooShortError)
+      lambda { @parser.parse }.should raise_error(Slither::LineWrongSizeError)
     end
 
+  end
+
+  describe "when parsing by bytes" do
+    before(:each) do
+      @definition = Slither.define :test, :by_bytes => true, :encoding => Encoding::UTF_8 do |d|
+        d.body do |b|
+          b.trap { true }
+          b.column :first, 5
+          b.column :last, 5
+          b.column :eol, 1
+        end   
+      end
+
+      @file_io = double("IO")
+      @file_io.stub(:external_encoding).and_return(Encoding::UTF_8)
+      @parser = Slither::Parser.new(@definition, @file_io)
+    end
+
+    it "should parse valid input" do
+      return_strings = ["abcdeABCDE\n","123  987  \n"].map{|str| str.encode! Encoding::ASCII_8BIT}
+      return_strings << nil
+      @file_io.should_receive(:read).exactly(3).times.with(11).and_return(return_strings[0],return_strings[1],return_strings[2])
+
+      expected = {
+        :body => [
+          {:first => 'abcde', :last => 'ABCDE', :eol => ''},
+          {:first => '123', :last => '987', :eol => ''}
+          ]
+      }
+
+      @parser.parse_by_bytes.should eq(expected)
+    end
+
+    it 'should raise error for data with incorrect length' do
+      return_strings = [ "abcdefghij".encode!(Encoding::ASCII_8BIT) ]
+      return_strings << nil
+      @file_io.should_receive(:read).with(11).and_return(return_strings[0])
+      lambda { @parser.parse_by_bytes }.should raise_error(Slither::LineWrongSizeError)
+    end
 
-    # it "raise an error if a section limit is over run"
+    it 'should handle utf characters' do
+      utf_str1 = "\xE5\x9B\xBD45"
+      utf_str2 = "ab\xE5\x9B\xBD"
+      return_strings = [(utf_str1 + utf_str2 + "\n").encode!(Encoding::ASCII_8BIT) ]
+      return_strings << nil
+      @file_io.should_receive(:read).exactly(2).times.with(11).and_return(return_strings[0],return_strings[1])
+
+      expected = {
+        :body => [ {:first => utf_str1, :last => utf_str2, :eol => ''} ]
+      }
+
+      @parser.parse_by_bytes.should eq(expected)
+    end
   end
 end

spec/slither_spec.rb

@@ -76,7 +76,7 @@
       File.stub!(:exists? => true)
       file_io = mock("IO")
       parser = mock("parser")
-      definition = mock('definition')
+      definition = Slither::Definition.new
 
       File.should_receive(:open).and_return(file_io)
       Slither.should_receive(:definition).with(:test).and_return(definition)