added checksum and do checksum

close #115
sisong · Jan 19, 2019 · b7d5305 · b7d5305
1 parent 7b46ec4
commit b7d5305
Show file tree

Hide file tree

Showing 18 changed files with 282 additions and 91 deletions.
diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@ HPATCH_OBJ := \
     dirDiffPatch/dir_patch/res_handle_limit.o \
     dirDiffPatch/dir_patch/ref_stream.o \
     dirDiffPatch/dir_patch/new_stream.o \
+    libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.o \
     file_for_patch.o
 
 HDIFF_OBJ := \
@@ -13,7 +14,6 @@ HDIFF_OBJ := \
     libHDiffPatch/HDiff/private_diff/compress_detect.o \
     libHDiffPatch/HDiff/private_diff/limit_mem_diff/digest_matcher.o \
     libHDiffPatch/HDiff/private_diff/limit_mem_diff/stream_serialize.o \
-    libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.o \
     libHDiffPatch/HDiff/private_diff/libdivsufsort/divsufsort64.o \
     libHDiffPatch/HDiff/private_diff/libdivsufsort/divsufsort.o \
     dirDiffPatch/dir_diff/dir_diff.o \

diff --git a/builds/codeblocks/HPatchZ.cbp b/builds/codeblocks/HPatchZ.cbp
@@ -118,6 +118,10 @@
 		<Unit filename="../../hpatchz.c">
 			<Option compilerVar="CC" />
 		</Unit>
+		<Unit filename="../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c">
+			<Option compilerVar="CC" />
+		</Unit>
+		<Unit filename="../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h" />
 		<Unit filename="../../libHDiffPatch/HPatch/patch.c">
 			<Option compilerVar="CC" />
 		</Unit>

diff --git a/builds/vc/HPatchZ.vcxproj b/builds/vc/HPatchZ.vcxproj
@@ -177,11 +177,13 @@
     <ClCompile Include="..\..\dirDiffPatch\dir_patch\res_handle_limit.c" />
     <ClCompile Include="..\..\file_for_patch.c" />
     <ClCompile Include="..\..\hpatchz.c" />
+    <ClCompile Include="..\..\libHDiffPatch\HDiff\private_diff\limit_mem_diff\adler_roll.c" />
     <ClCompile Include="..\..\libHDiffPatch\HPatch\patch.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\dirDiffPatch\file_for_dirPatch.h" />
     <ClInclude Include="..\..\file_for_patch.h" />
+    <ClInclude Include="..\..\libHDiffPatch\HDiff\private_diff\limit_mem_diff\adler_roll.h" />
     <ClInclude Include="..\..\libHDiffPatch\HPatch\patch.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

diff --git a/builds/xcode/hpatchz.xcodeproj/project.pbxproj b/builds/xcode/hpatchz.xcodeproj/project.pbxproj
@@ -14,6 +14,7 @@
 		D64B4BFB2077B91A0039C38B /* libbz2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = D64B4BFA2077B91A0039C38B /* libbz2.tbd */; };
 		D67D3BF421D3C5870050632A /* ref_stream.c in Sources */ = {isa = PBXBuildFile; fileRef = D67D3BF121D3C5870050632A /* ref_stream.c */; };
 		D67D3BF521D3C5870050632A /* dir_patch.c in Sources */ = {isa = PBXBuildFile; fileRef = D67D3BF221D3C5870050632A /* dir_patch.c */; };
+		D6BAF70321F2F61B00C30C0A /* adler_roll.c in Sources */ = {isa = PBXBuildFile; fileRef = D6BAF70121F2F61B00C30C0A /* adler_roll.c */; };
 		D6BD9BCE17578601004886DE /* patch.c in Sources */ = {isa = PBXBuildFile; fileRef = D6BD9BCA17578601004886DE /* patch.c */; };
 		D6BF3A5721E8CDC300833537 /* res_handle_limit.c in Sources */ = {isa = PBXBuildFile; fileRef = D6BF3A5621E8CDC300833537 /* res_handle_limit.c */; };
 		D6BF3A5921E95F4200833537 /* file_for_patch.c in Sources */ = {isa = PBXBuildFile; fileRef = D6BF3A5821E95F4200833537 /* file_for_patch.c */; };
@@ -52,6 +53,10 @@
 		D67D3BF221D3C5870050632A /* dir_patch.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dir_patch.c; sourceTree = "<group>"; };
 		D67D3BF321D3C5870050632A /* ref_stream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ref_stream.h; sourceTree = "<group>"; };
 		D67D3BF621D48DCB0050632A /* patch_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = patch_private.h; sourceTree = "<group>"; };
+		D6BAF6F221F2E08000C30C0A /* checksum_plugin.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = checksum_plugin.h; sourceTree = "<group>"; };
+		D6BAF6F521F2E34600C30C0A /* checksum_plugin_demo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = checksum_plugin_demo.h; path = ../../checksum_plugin_demo.h; sourceTree = "<group>"; };
+		D6BAF70121F2F61B00C30C0A /* adler_roll.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = adler_roll.c; path = ../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c; sourceTree = "<group>"; };
+		D6BAF70221F2F61B00C30C0A /* adler_roll.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = adler_roll.h; path = ../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h; sourceTree = "<group>"; };
 		D6BD9BCA17578601004886DE /* patch.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = patch.c; sourceTree = "<group>"; };
 		D6BF3A5521E8CDC300833537 /* res_handle_limit.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = res_handle_limit.h; sourceTree = "<group>"; };
 		D6BF3A5621E8CDC300833537 /* res_handle_limit.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = res_handle_limit.c; sourceTree = "<group>"; };
@@ -93,10 +98,12 @@
 		D6553CAA165FA1A9009D4C44 = {
 			isa = PBXGroup;
 			children = (
+				D6BAF70021F2F60500C30C0A /* adler_roll */,
 				D67D3BEF21D3C5870050632A /* dir_patch */,
 				D6BD9BC917578601004886DE /* HPatch */,
 				D6241AD021DC6B6900E2846A /* lzma */,
 				D64B4BF32077AB550039C38B /* decompress_plugin_demo.h */,
+				D6BAF6F521F2E34600C30C0A /* checksum_plugin_demo.h */,
 				D64B4BF22077AB550039C38B /* _clock_for_demo.h */,
 				D639ACD9208334A800811BB3 /* _atosize.h */,
 				D6BF3A5821E95F4200833537 /* file_for_patch.c */,
@@ -133,9 +140,19 @@
 			path = ../../dirDiffPatch/dir_patch;
 			sourceTree = "<group>";
 		};
+		D6BAF70021F2F60500C30C0A /* adler_roll */ = {
+			isa = PBXGroup;
+			children = (
+				D6BAF70121F2F61B00C30C0A /* adler_roll.c */,
+				D6BAF70221F2F61B00C30C0A /* adler_roll.h */,
+			);
+			name = adler_roll;
+			sourceTree = "<group>";
+		};
 		D6BD9BC917578601004886DE /* HPatch */ = {
 			isa = PBXGroup;
 			children = (
+				D6BAF6F221F2E08000C30C0A /* checksum_plugin.h */,
 				D67D3BF621D48DCB0050632A /* patch_private.h */,
 				D64B4BF02077AB290039C38B /* patch_types.h */,
 				D64B4BF12077AB2A0039C38B /* patch.h */,
@@ -203,6 +220,7 @@
 				D6BF3A5721E8CDC300833537 /* res_handle_limit.c in Sources */,
 				D6241AE521DE47EC00E2846A /* new_stream.c in Sources */,
 				D6BF3A5921E95F4200833537 /* file_for_patch.c in Sources */,
+				D6BAF70321F2F61B00C30C0A /* adler_roll.c in Sources */,
 				D64B4BEF2077AAAA0039C38B /* hpatchz.c in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

diff --git a/checksum_plugin_demo.h b/checksum_plugin_demo.h
@@ -28,15 +28,14 @@
 #ifndef HPatch_checksum_plugin_demo_h
 #define HPatch_checksum_plugin_demo_h
 //checksum plugin demo:
-//  crc32ChecksumPlugin     =  32 bit effective
-//  adler32ChecksumPlugin   ~  29 bit effective
-//  adler64ChecksumPlugin   ~  36 bit effective
-//  adler32fChecksumPlugin  ~  32 bit effective
-//  adler64fChecksumPlugin  ~  63 bit effective
-//  md5ChecksumPlugin       ? 128 bit effective
+//  crc32ChecksumPlugin
+//  adler32ChecksumPlugin
+//  adler64ChecksumPlugin
+//  adler32fChecksumPlugin
+//  adler64fChecksumPlugin
+//  md5ChecksumPlugin
 
 #include "libHDiffPatch/HPatch/checksum_plugin.h"
-#include "libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h"
 
 #ifndef _IsNeedIncludeDefaultChecksumHead
 #   define _IsNeedIncludeDefaultChecksumHead 1
@@ -57,11 +56,11 @@
 #if (_IsNeedIncludeDefaultChecksumHead)
 #   include "zlib.h" // http://zlib.net/  https://github.com/madler/zlib
 #endif
-static const char* _crc32_checksumType(){
+static const char* _crc32_checksumType(void){
     static const char* type="crc32";
     return type;
 }
-static size_t _crc32_checksumByteSize(){
+static size_t _crc32_checksumByteSize(void){
     return sizeof(hpatch_uint32_t);
 }
 static hpatch_checksumHandle _crc32_open(hpatch_TChecksum* plugin){
@@ -100,11 +99,15 @@ static hpatch_TChecksum crc32ChecksumPlugin={ _crc32_checksumType,_crc32_checksu
 #endif//_ChecksumPlugin_crc32
 
 
-static const char* _adler32_checksumType(){
+#ifdef  _ChecksumPlugin_adler32
+#   ifndef _ChecksumPlugin_crc32
+#       include "libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h"
+#   endif
+static const char* _adler32_checksumType(void){
     static const char* type="adler32";
     return type;
 }
-static size_t _adler32_checksumByteSize(){
+static size_t _adler32_checksumByteSize(void){
     return sizeof(hpatch_uint32_t);
 }
 static hpatch_checksumHandle _adler32_open(hpatch_TChecksum* plugin){
@@ -148,13 +151,16 @@ static void _adler32_end(hpatch_checksumHandle handle,
 }
 static hpatch_TChecksum adler32ChecksumPlugin={ _adler32_checksumType,_adler32_checksumByteSize,_adler32_open,
                                                 _adler32_close,_adler32_begin,_adler32_append,_adler32_end};
+#endif//_ChecksumPlugin_adler32
 
 
-static const char* _adler64_checksumType(){
+#ifdef  _ChecksumPlugin_adler64
+#include "libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h"
+static const char* _adler64_checksumType(void){
     static const char* type="adler64";
     return type;
 }
-static size_t _adler64_checksumByteSize(){
+static size_t _adler64_checksumByteSize(void){
     return sizeof(hpatch_uint64_t);
 }
 static hpatch_checksumHandle _adler64_open(hpatch_TChecksum* plugin){
@@ -180,13 +186,16 @@ static void _adler64_end(hpatch_checksumHandle handle,
 }
 static hpatch_TChecksum adler64ChecksumPlugin={ _adler64_checksumType,_adler64_checksumByteSize,_adler64_open,
                                                 _adler64_close,_adler64_begin,_adler64_append,_adler64_end};
+#endif//_ChecksumPlugin_adler64
 
 
-static const char* _adler32f_checksumType(){
+#ifdef  _ChecksumPlugin_adler32f
+#include "libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h"
+static const char* _adler32f_checksumType(void){
     static const char* type="adler32f";
     return type;
 }
-static size_t _adler32f_checksumByteSize(){
+static size_t _adler32f_checksumByteSize(void){
     return sizeof(hpatch_uint32_t);
 }
 static hpatch_checksumHandle _adler32f_open(hpatch_TChecksum* plugin){
@@ -212,13 +221,16 @@ static void _adler32f_end(hpatch_checksumHandle handle,
 }
 static hpatch_TChecksum adler32fChecksumPlugin={ _adler32f_checksumType,_adler32f_checksumByteSize,_adler32f_open,
                                                  _adler32f_close,_adler32f_begin,_adler32f_append,_adler32f_end};
+#endif//_ChecksumPlugin_adler32f
 
 
-static const char* _adler64f_checksumType(){
+#ifdef  _ChecksumPlugin_adler64f
+#include "libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.h"
+static const char* _adler64f_checksumType(void){
     static const char* type="adler64f";
     return type;
 }
-static size_t _adler64f_checksumByteSize(){
+static size_t _adler64f_checksumByteSize(void){
     return sizeof(hpatch_uint64_t);
 }
 static hpatch_checksumHandle _adler64f_open(hpatch_TChecksum* plugin){
@@ -244,17 +256,18 @@ static void _adler64f_end(hpatch_checksumHandle handle,
 }
 static hpatch_TChecksum adler64fChecksumPlugin={ _adler64f_checksumType,_adler64f_checksumByteSize,_adler64f_open,
                                                  _adler64f_close,_adler64f_begin,_adler64f_append,_adler64f_end};
+#endif//_ChecksumPlugin_adler64f
 
 
 #ifdef  _ChecksumPlugin_md5
 #if (_IsNeedIncludeDefaultChecksumHead)
 #   include "md5.h" // https://sourceforge.net/projects/libmd5-rfc
 #endif
-static const char* _md5_checksumType(){
+static const char* _md5_checksumType(void){
     static const char* type="md5";
     return type;
 }
-static size_t _md5_checksumByteSize(){
+static size_t _md5_checksumByteSize(void){
     return 16;
 }
 static hpatch_checksumHandle _md5_open(hpatch_TChecksum* plugin){

diff --git a/compress_plugin_demo.h b/compress_plugin_demo.h
@@ -92,11 +92,11 @@ static size_t _fun_compress_name(const hdiff_TCompress* compressPlugin, \
 #   include "zlib.h" // http://zlib.net/  https://github.com/madler/zlib
 #endif
     static int zlib_compress_level=9; //1..9
-    static const char*  _zlib_stream_compressType(){
+    static const char*  _zlib_stream_compressType(void){
         static const char* kCompressType="zlib";
         return kCompressType;
     }
-    static const char*  _zlib_compressType(){
+    static const char*  _zlib_compressType(void){
         return _zlib_stream_compressType();
     }
     static size_t  _zlib_maxCompressedSize(const hdiff_TCompress* compressPlugin,size_t dataSize){
@@ -240,11 +240,11 @@ static size_t _fun_compress_name(const hdiff_TCompress* compressPlugin, \
 #   include "bzlib.h" // http://www.bzip.org/
 #endif
     static int bz2_compress_level=9; //1..9
-    static const char*  _bz2_stream_compressType(){
+    static const char*  _bz2_stream_compressType(void){
         static const char* kCompressType="bz2";
         return kCompressType;
     }
-    static const char*  _bz2_compressType(){
+    static const char*  _bz2_compressType(void){
         return _bz2_stream_compressType();
     }
     static size_t  _bz2_maxCompressedSize(const hdiff_TCompress* compressPlugin,size_t dataSize){
@@ -324,11 +324,11 @@ static size_t _fun_compress_name(const hdiff_TCompress* compressPlugin, \
 #endif
     static int lzma_compress_level=7;//0..9
     static UInt32 lzma_dictSize=1<<22;  //patch decompress need 4*lzma_dictSize memroy
-    static const char*  _lzma_stream_compressType(){
+    static const char*  _lzma_stream_compressType(void){
         static const char* kCompressType="lzma";
         return kCompressType;
     }
-    static const char*  _lzma_compressType(){
+    static const char*  _lzma_compressType(void){
         return _lzma_stream_compressType();
     }
     static size_t  _lzma_maxCompressedSize(const hdiff_TCompress* compressPlugin,size_t dataSize){
@@ -451,11 +451,11 @@ static size_t _fun_compress_name(const hdiff_TCompress* compressPlugin, \
 #   include "lz4hc.h"
 #endif
     static int lz4hc_compress_level=11; //3..12
-    static const char*  _lz4_stream_compressType(){
+    static const char*  _lz4_stream_compressType(void){
         static const char* kCompressType="lz4";
         return kCompressType;
     }
-    static const char*  _lz4_compressType(){
+    static const char*  _lz4_compressType(void){
         return _lz4_stream_compressType();
     }
     static size_t  _lz4_maxCompressedSize(const hdiff_TCompress* compressPlugin,size_t dataSize){
@@ -599,11 +599,11 @@ static size_t _fun_compress_name(const hdiff_TCompress* compressPlugin, \
 #   include "zstd.h" // "zstd/lib/zstd.h" https://github.com/facebook/zstd
 #endif
     static int zstd_compress_level=20; //0..22
-    static const char*  _zstd_stream_compressType(){
+    static const char*  _zstd_stream_compressType(void){
         static const char* kCompressType="zstd";
         return kCompressType;
     }
-    static const char*  _zstd_compressType(){
+    static const char*  _zstd_compressType(void){
         return _zstd_stream_compressType();
     }
     static size_t  _zstd_maxCompressedSize(const hdiff_TCompress* compressPlugin,size_t dataSize){

diff --git a/decompress_plugin_demo.h b/decompress_plugin_demo.h
@@ -35,6 +35,7 @@
 //  zstdDecompressPlugin;
 
 #include <stdlib.h> //malloc free
+#include <stdio.h>  //fprintf
 #include "libHDiffPatch/HPatch/patch_types.h"
 
 #define kDecompressBufSize (1024*16)

diff --git a/dirDiffPatch/dir_diff/dir_diff.cpp b/dirDiffPatch/dir_diff/dir_diff.cpp
@@ -27,6 +27,7 @@
  OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "dir_diff.h"
+#include <stdio.h>
 #include <algorithm> //sort
 #include <map>
 #include <set>
@@ -530,9 +531,6 @@ void dir_diff(IDirDiffListener* listener,const std::string& oldPath,const std::s
     CRefStream newRefStream;
     oldRefStream.open(resLimit.limit.streamList,oldRefIList.size());
     newRefStream.open(resLimit.limit.streamList+oldRefIList.size(),newRefIList.size());
-    listener->diffRefInfo(oldList.size(),newList.size(),dataSamePairList.size(),
-                          oldRefIList.size(),newRefIList.size(),
-                          oldRefStream.stream->streamSize,newRefStream.stream->streamSize);
 
     //checksum
     const size_t checksumByteSize=(checksumPlugin==0)?0:checksumPlugin->checksumByteSize();
@@ -543,11 +541,17 @@ void dir_diff(IDirDiffListener* listener,const std::string& oldPath,const std::s
     oldRefChecksum.appendEnd();
     newRefChecksum.append(newRefStream.stream);
     newRefChecksum.appendEnd();
+    hpatch_StreamPos_t sameFileSize=0;
     for (size_t i=0; i<dataSamePairList.size(); ++i) {
         CFileStreamInput file(newList[dataSamePairList[i].newIndex]);
         sameFileChecksum.append(&file.base);
+        sameFileSize+=file.base.streamSize;
     }
     sameFileChecksum.appendEnd();
+
+    listener->diffRefInfo(oldList.size(),newList.size(),dataSamePairList.size(),
+                          sameFileSize,oldRefIList.size(),newRefIList.size(),
+                          oldRefStream.stream->streamSize,newRefStream.stream->streamSize);
 
     //serialize headData
     std::vector<TByte> headData;
@@ -583,6 +587,7 @@ void dir_diff(IDirDiffListener* listener,const std::string& oldPath,const std::s
     packUInt(out_data,oldRefIList.size());      clearVector(oldRefIList);
     packUInt(out_data,newRefIList.size());      clearVector(newRefIList);
     packUInt(out_data,dataSamePairList.size()); clearVector(dataSamePairList);
+    packUInt(out_data,sameFileSize);
     //externData size
     std::vector<TByte> externData;
     listener->externData(externData);
@@ -686,9 +691,9 @@ struct CDirPatchListener:public IDirPatchListener{
         std::string path(_path);
         if (path.empty()) return path;
         if (path[path.size()-1]==kPatch_dirSeparator)
-            path.pop_back();
+            path.resize(path.size()-1);
         while ((!path.empty())&&(path[path.size()-1]!=kPatch_dirSeparator)) {
-            path.pop_back();
+            path.resize(path.size()-1);
         }
         return path;
     }

diff --git a/dirDiffPatch/dir_diff/dir_diff.h b/dirDiffPatch/dir_diff/dir_diff.h
@@ -50,7 +50,7 @@ struct IDirDiffListener:public IDirFilter{
     virtual ~IDirDiffListener(){}
     virtual void diffPathList(std::vector<std::string>& oldPathList,std::vector<std::string>& newPathList){}
     virtual void diffRefInfo(size_t oldPathCount,size_t newPathCount,size_t sameFilePairCount,
-                             size_t refOldFileCount,size_t refNewFileCount,
+                             hpatch_StreamPos_t sameFileSize,size_t refOldFileCount,size_t refNewFileCount,
                              hpatch_StreamPos_t refOldFileSize,hpatch_StreamPos_t refNewFileSize){}
     virtual void runHDiffBegin(){}
     virtual void runHDiffEnd(hpatch_StreamPos_t diffDataSize){}