Fix ossrs#1694, Support DVR 2GB+ MP4 file. 3.0.155

README.md

@@ -145,6 +145,7 @@ For previous versions, please read:
 
 ## V3 changes
 
+* v3.0, 2020-12-17, Fix [#1694][bug #1694], Support DVR 2GB+ MP4 file. 3.0.155
 * v3.0, 2020-12-17, Fix [#1548][bug #1548], Add edts in MP4 for Windows10. 3.0.154
 * <strong>v3.0, 2020-10-31, [3.0 release2(3.0.153)][r3.0r2] released. 122663 lines.</strong>
 * v3.0, 2020-10-31, Fix [#509][bug #509], Always malloc stack on heap. 3.0.153
@@ -1709,6 +1710,7 @@ Winlin
 [bug #1780]: https://github.com/ossrs/srs/issues/1780
 [bug #1987]: https://github.com/ossrs/srs/issues/1987
 [bug #1548]: https://github.com/ossrs/srs/issues/1548
+[bug #1694]: https://github.com/ossrs/srs/issues/1694
 [bug #yyyyyyyyyyyyy]: https://github.com/ossrs/srs/issues/yyyyyyyyyyyyy
 
 [exo #828]: https://github.com/google/ExoPlayer/pull/828

trunk/src/core/srs_core_version3.hpp

@@ -24,6 +24,6 @@
 #ifndef SRS_CORE_VERSION3_HPP
 #define SRS_CORE_VERSION3_HPP
 
-#define SRS_VERSION3_REVISION 154
+#define SRS_VERSION3_REVISION 155
 
 #endif

trunk/src/kernel/srs_kernel_buffer.hpp

@@ -62,19 +62,12 @@ class ISrsCodec
     ISrsCodec();
     virtual ~ISrsCodec();
 public:
-    /**
-     * get the number of bytes to code to.
-     */
-    // TODO: FIXME: change to uint64_t.
-    virtual int nb_bytes() = 0;
-    /**
-     * encode object to bytes in SrsBuffer.
-     */
+    // Get the number of bytes to code to.
+    virtual uint64_t nb_bytes() = 0;
+    // Encode object to bytes in SrsBuffer.
     virtual srs_error_t encode(SrsBuffer* buf) = 0;
 public:
-    /**
-     * decode object from bytes in SrsBuffer.
-     */
+    // Decode object from bytes in SrsBuffer.
     virtual srs_error_t decode(SrsBuffer* buf) = 0;
 };
 

trunk/src/kernel/srs_kernel_error.hpp

@@ -287,6 +287,7 @@
 #define ERROR_INOTIFY_CREATE                3092
 #define ERROR_INOTIFY_OPENFD                3093
 #define ERROR_INOTIFY_WATCH                 3094
+#define ERROR_MP4_ILLEGAL_MDAT              3090
 
 ///////////////////////////////////////////////////////
 // HTTP/StreamCaster protocol error.

trunk/src/kernel/srs_kernel_mp4.cpp

@@ -190,17 +190,18 @@ int SrsMp4Box::sz_header()
     return nb_header();
 }
 
-int SrsMp4Box::update_size()
+uint64_t SrsMp4Box::update_size()
 {
     uint64_t size = nb_bytes();
 
     if (size > 0xffffffff) {
         largesize = size;
+        smallsize = SRS_MP4_USE_LARGE_SIZE;
     } else {
         smallsize = (uint32_t)size;
     }
 
-    return (int)size;
+    return size;
 }
 
 int SrsMp4Box::left_space(SrsBuffer* buf)
@@ -299,11 +300,11 @@ srs_error_t SrsMp4Box::discovery(SrsBuffer* buf, SrsMp4Box** ppbox)
     *ppbox = NULL;
 
     srs_error_t err = srs_success;
-    
+
     if (!buf->require(8)) {
         return srs_error_new(ERROR_MP4_BOX_REQUIRE_SPACE, "requires 8 only %d bytes", buf->left());
     }
-    
+
     // Discovery the size and type.
     uint64_t largesize = 0;
     uint32_t smallsize = (uint32_t)buf->read_4bytes();
@@ -313,15 +314,14 @@ srs_error_t SrsMp4Box::discovery(SrsBuffer* buf, SrsMp4Box** ppbox)
             return srs_error_new(ERROR_MP4_BOX_REQUIRE_SPACE, "requires 16 only %d bytes", buf->left());
         }
         largesize = (uint64_t)buf->read_8bytes();
-        buf->skip(-8);
     }
+
+    // Reset the buffer, because we only peek it.
     buf->skip(-8);
-
-    // Only support 31bits size.
-    if (largesize > 0x7fffffff) {
-        return srs_error_new(ERROR_MP4_BOX_OVERFLOW, "overflow 31bits, largesize=%" PRId64, largesize);
+    if (smallsize == SRS_MP4_USE_LARGE_SIZE) {
+        buf->skip(-8);
     }
-    
+
     SrsMp4Box* box = NULL;
     switch(type) {
         case SrsMp4BoxTypeFTYP: box = new SrsMp4FileTypeBox(); break;
@@ -382,9 +382,9 @@ srs_error_t SrsMp4Box::discovery(SrsBuffer* buf, SrsMp4Box** ppbox)
     return err;
 }
 
-int SrsMp4Box::nb_bytes()
+uint64_t SrsMp4Box::nb_bytes()
 {
-    int sz = nb_header();
+    uint64_t sz = nb_header();
 
     vector<SrsMp4Box*>::iterator it;
     for (it = boxes.begin(); it != boxes.end(); ++it) {
@@ -488,21 +488,16 @@ srs_error_t SrsMp4Box::encode_header(SrsBuffer* buf)
 {
     srs_error_t err = srs_success;
 
-    // Only support 31bits size.
-    if (sz() > 0x7fffffff) {
-        return srs_error_new(ERROR_MP4_BOX_OVERFLOW, "box size overflow 31bits, size=%" PRId64, sz());
-    }
-
     int size = SrsMp4Box::nb_header();
     if (!buf->require(size)) {
         return srs_error_new(ERROR_MP4_BOX_REQUIRE_SPACE, "requires %d only %d bytes", size, buf->left());
     }
 
     buf->write_4bytes(smallsize);
+    buf->write_4bytes(type);
     if (smallsize == SRS_MP4_USE_LARGE_SIZE) {
         buf->write_8bytes(largesize);
     }
-    buf->write_4bytes(type);
 
     if (type == SrsMp4BoxTypeUUID) {
         buf->write_bytes(&usertype[0], 16);
@@ -1071,7 +1066,7 @@ SrsMp4TrunEntry::~SrsMp4TrunEntry()
 {
 }
 
-int SrsMp4TrunEntry::nb_bytes()
+uint64_t SrsMp4TrunEntry::nb_bytes()
 {
     int size = 0;
 
@@ -1291,7 +1286,7 @@ SrsMp4MediaDataBox::~SrsMp4MediaDataBox()
 {
 }
 
-int SrsMp4MediaDataBox::nb_bytes()
+uint64_t SrsMp4MediaDataBox::nb_bytes()
 {
     return SrsMp4Box::nb_header() + nb_data;
 }
@@ -1315,7 +1310,7 @@ srs_error_t SrsMp4MediaDataBox::decode(SrsBuffer* buf)
         return srs_error_wrap(err, "decode box");
     }
 
-    nb_data = (int)(sz() - nb_header());
+    nb_data = sz() - (uint64_t)nb_header();
 
     return err;
 }
@@ -3270,7 +3265,7 @@ int SrsMp4BaseDescriptor::left_space(SrsBuffer* buf)
     return srs_max(0, left);
 }
 
-int SrsMp4BaseDescriptor::nb_bytes()
+uint64_t SrsMp4BaseDescriptor::nb_bytes()
 {
     // 1 byte tag.
     int size = 1;
@@ -5962,29 +5957,41 @@ srs_error_t SrsMp4Encoder::flush()
 
     // Write mdat box.
     if (true) {
-        // Update the mdat box header.
-        if ((err = wsio->lseek(mdat_offset, SEEK_SET, NULL)) != srs_success) {
-            return srs_error_wrap(err, "seek to mdat");
-        }
-
         // Write mdat box with size of data,
         // its payload already writen by samples,
         // and we will update its header(size) when flush.
         SrsMp4MediaDataBox* mdat = new SrsMp4MediaDataBox();
         SrsAutoFree(SrsMp4MediaDataBox, mdat);
+
+        // Update the size of mdat first, for over 2GB file.
+        mdat->nb_data = mdat_bytes;
+        mdat->update_size();
 
         int nb_data = mdat->sz_header();
         uint8_t* data = new uint8_t[nb_data];
         SrsAutoFreeA(uint8_t, data);
 
         SrsBuffer* buffer = new SrsBuffer((char*)data, nb_data);
         SrsAutoFree(SrsBuffer, buffer);
-
-        // TODO: FIXME: Support 64bits size.
-        mdat->nb_data = (int)mdat_bytes;
+
         if ((err = mdat->encode(buffer)) != srs_success) {
             return srs_error_wrap(err, "encode mdat");
         }
+
+        // We might adjust the offset of mdat, for large size, 2GB+ as such.
+        if (nb_data > 8) {
+            // For large size, the header of mdat MUST be 16.
+            if (nb_data != 16) {
+                return srs_error_new(ERROR_MP4_ILLEGAL_MDAT, "Invalid mdat header size %d", nb_data);
+            }
+            // Use large size, to the start of reserved free box.
+            mdat_offset -= 8;
+        }
+
+        // Seek to the start of mdat.
+        if ((err = wsio->lseek(mdat_offset, SEEK_SET, NULL)) != srs_success) {
+            return srs_error_wrap(err, "seek to mdat");
+        }
 
         // TODO: FIXME: Ensure all bytes are writen.
         if ((err = wsio->write(data, nb_data, NULL)) != srs_success) {
@@ -6447,7 +6454,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts)
         // @remark Remember the data_offset of turn is size(moof)+header(mdat), not including styp or sidx.
         int moof_bytes = moof->nb_bytes();
         trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header());
-        mdat->nb_data = (int)mdat_bytes;
+        mdat->nb_data = mdat_bytes;
 
         // Update the size of sidx.
         SrsMp4SegmentIndexEntry* entry = &sidx->entries[0];

trunk/src/kernel/srs_kernel_mp4.hpp

@@ -205,7 +205,7 @@ class SrsMp4Box : public ISrsCodec
     // @remark For mdat box, we must codec its header, use this instead of sz().
     virtual int sz_header();
     // Update the size of box.
-    virtual int update_size();
+    virtual uint64_t update_size();
     // Get the left space of box, for decoder.
     virtual int left_space(SrsBuffer* buf);
     // Box type helper.
@@ -227,7 +227,7 @@ class SrsMp4Box : public ISrsCodec
     static srs_error_t discovery(SrsBuffer* buf, SrsMp4Box** ppbox);
 // Interface ISrsCodec
 public:
-    virtual int nb_bytes();
+    virtual uint64_t nb_bytes();
     virtual srs_error_t encode(SrsBuffer* buf);
     virtual srs_error_t decode(SrsBuffer* buf);
 protected:
@@ -495,7 +495,7 @@ class SrsMp4TrunEntry : public ISrsCodec
     SrsMp4TrunEntry(SrsMp4FullBox* o);
     virtual ~SrsMp4TrunEntry();
 
-    virtual int nb_bytes();
+    virtual uint64_t nb_bytes();
     virtual srs_error_t encode(SrsBuffer* buf);
     virtual srs_error_t decode(SrsBuffer* buf);
 
@@ -581,16 +581,15 @@ class SrsMp4MediaDataBox : public SrsMp4Box
 {
 public:
     // The contained media data, which we never directly read/write it.
-    // TODO: FIXME: Support 64bits size.
-    int nb_data;
+    uint64_t nb_data;
 public:
     SrsMp4MediaDataBox();
     virtual ~SrsMp4MediaDataBox();
 // Interface ISrsCodec
 public:
     // The total size of bytes, including the sz_header() and nb_data,
     // which used to write the smallsize or largesize of mp4.
-    virtual int nb_bytes();
+    virtual uint64_t nb_bytes();
     // To encode the mdat box, the buf should only contains the sz_header(),
     // because the mdata only encode the header.
     virtual srs_error_t encode(SrsBuffer* buf);
@@ -1369,7 +1368,7 @@ class SrsMp4BaseDescriptor : public ISrsCodec
     virtual int left_space(SrsBuffer* buf);
 // Interface ISrsCodec
 public:
-    virtual int nb_bytes();
+    virtual uint64_t nb_bytes();
     virtual srs_error_t encode(SrsBuffer* buf);
     virtual srs_error_t decode(SrsBuffer* buf);
 protected:

trunk/src/utest/srs_utest_kernel.hpp

@@ -130,7 +130,7 @@ class MockSrsCodec : public ISrsCodec
     MockSrsCodec();
     virtual ~MockSrsCodec();
 public:
-    virtual int nb_bytes();
+    virtual uint64_t nb_bytes();
     virtual srs_error_t encode(SrsBuffer* buf);
     virtual srs_error_t decode(SrsBuffer* buf);
 };