From 798f1db039ed9eabb12eba1b1151d379591b79f3 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sun, 10 Jan 2016 21:28:07 -0800 Subject: [PATCH] h264.cc: handle both kinds of ffmpeg extradata. --- src/h264-test.cc | 51 ++++++++++++++++-------- src/h264.cc | 100 +++++++++++++++++++++++++++-------------------- src/h264.h | 23 ++++++----- 3 files changed, 106 insertions(+), 68 deletions(-) diff --git a/src/h264-test.cc b/src/h264-test.cc index 5411b81..1cf5638 100644 --- a/src/h264-test.cc +++ b/src/h264-test.cc @@ -43,15 +43,32 @@ DECLARE_bool(alsologtostderr); namespace moonfire_nvr { namespace { -const uint8_t kTestInput[] = { +const uint8_t kAnnexBTestInput[] = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80}; +const uint8_t kAvcDecoderConfigTestInput[] = { + 0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d, + 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, + 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, + 0x01, 0x01, 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80}; + +const char kTestOutput[] = + "00 00 00 84 61 76 63 31 00 00 00 00 00 00 00 01 " + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " + "05 00 02 d0 00 48 00 00 00 48 00 00 00 00 00 00 " + "00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " + "00 00 00 18 ff ff 00 00 00 2e 61 76 63 43 01 4d " + "00 1f ff e1 00 17 67 4d 00 1f 9a 66 02 80 2d ff " + "35 01 01 01 40 00 00 fa 00 00 1d 4c 01 01 00 04 " + "68 ee 3c 80"; + TEST(H264Test, DecodeOnly) { std::vector nal_units_hexed; - re2::StringPiece test_input(reinterpret_cast(kTestInput), - sizeof(kTestInput)); + re2::StringPiece test_input(reinterpret_cast(kAnnexBTestInput), + sizeof(kAnnexBTestInput)); internal::NalUnitFunction fn = [&nal_units_hexed](re2::StringPiece nal_unit) { nal_units_hexed.push_back(ToHex(nal_unit)); return IterationControl::kContinue; @@ -65,20 +82,22 @@ TEST(H264Test, DecodeOnly) { "68 ee 3c 80")); } -TEST(H264Test, SampleData) { - const char kTestOutput[] = - "00 00 00 84 61 76 63 31 00 00 00 00 00 00 00 01 " - "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "05 00 02 d0 00 48 00 00 00 48 00 00 00 00 00 00 " - "00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "00 00 00 18 ff ff 00 00 00 2e 61 76 63 43 01 4d " - "00 1f ff e1 00 17 67 4d 00 1f 9a 66 02 80 2d ff " - "35 01 01 01 40 00 00 fa 00 00 1d 4c 01 01 00 04 " - "68 ee 3c 80"; +TEST(H264Test, SampleDataFromAnnexBExtraData) { + re2::StringPiece test_input(reinterpret_cast(kAnnexBTestInput), + sizeof(kAnnexBTestInput)); + std::string sample_entry; + std::string error_message; + ASSERT_TRUE( + GetH264SampleEntry(test_input, 1280, 720, &sample_entry, &error_message)) + << error_message; - re2::StringPiece test_input(reinterpret_cast(kTestInput), - sizeof(kTestInput)); + EXPECT_EQ(kTestOutput, ToHex(sample_entry)); +} + +TEST(H264Test, SampleDataFromAvcDecoderConfigExtraData) { + re2::StringPiece test_input( + reinterpret_cast(kAvcDecoderConfigTestInput), + sizeof(kAvcDecoderConfigTestInput)); std::string sample_entry; std::string error_message; ASSERT_TRUE( diff --git a/src/h264.cc b/src/h264.cc index 3258cce..411677a 100644 --- a/src/h264.cc +++ b/src/h264.cc @@ -48,8 +48,8 @@ const int kNalUnitPicParameterSet = 8; // Parse sequence parameter set and picture parameter set from ffmpeg's // "extra_data". -bool ParseExtraData(re2::StringPiece extra_data, re2::StringPiece *sps, - re2::StringPiece *pps, std::string *error_message) { +bool ParseAnnexBExtraData(re2::StringPiece extradata, re2::StringPiece *sps, + re2::StringPiece *pps, std::string *error_message) { bool ok = true; internal::NalUnitFunction fn = [&ok, sps, pps, error_message](re2::StringPiece nal_unit) { @@ -70,7 +70,7 @@ bool ParseExtraData(re2::StringPiece extra_data, re2::StringPiece *sps, } return IterationControl::kContinue; }; - if (!internal::DecodeH264AnnexB(extra_data, fn, error_message) || !ok) { + if (!internal::DecodeH264AnnexB(extradata, fn, error_message) || !ok) { return false; } if (sps->empty() || pps->empty()) { @@ -92,7 +92,8 @@ bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit, static const RE2 kStartCode("(\\x00{2,}\\x01)"); if (!RE2::Consume(&data, kStartCode)) { - *error_message = "stream does not start with Annex B start code"; + *error_message = + StrCat("stream does not start with Annex B start code: ", ToHex(data)); return false; } @@ -124,18 +125,27 @@ bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit, } // namespace internal -bool GetH264SampleEntry(re2::StringPiece extra_data, uint16_t width, +bool GetH264SampleEntry(re2::StringPiece extradata, uint16_t width, uint16_t height, std::string *out, std::string *error_message) { + uint32_t avcc_len; re2::StringPiece sps; re2::StringPiece pps; - if (!ParseExtraData(extra_data, &sps, &pps, error_message)) { - return false; + if (extradata.starts_with(re2::StringPiece("\x00\x00\x00\x01", 4)) || + extradata.starts_with(re2::StringPiece("\x00\x00\x01", 3))) { + // ffmpeg supplied "extradata" in Annex B format. + if (!ParseAnnexBExtraData(extradata, &sps, &pps, error_message)) { + return false; + } + + // This magic value is checked at the end. + avcc_len = 19 + sps.size() + pps.size(); + } else { + // Assume "extradata" holds an AVCDecoderConfiguration. + avcc_len = 8 + extradata.size(); } - // These match the size of all fields below. - // Don't panic; they're verified at the end. - uint32_t avcc_len = 19 + sps.size() + pps.size(); + // This magic value is also checked at the end. uint32_t avc1_len = 86 + avcc_len; out->clear(); @@ -167,42 +177,48 @@ bool GetH264SampleEntry(re2::StringPiece extra_data, uint16_t width, AppendU32(avcc_len, out); // length out->append("avcC"); // type - // AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1. - // The beginning of the AVCDecoderConfiguration takes a few values from - // the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section - // defines the syntax in terms of RBSP, not NAL. The difference is the - // escaping of 00 00 01 and 00 00 02; see notes about - // "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4. - // It looks like 00 is not a valid value of profile_idc, so this distinction - // shouldn't be relevant here. And ffmpeg seems to ignore it. - out->push_back(1); // configurationVersion - out->push_back(sps[1]); // profile_idc -> AVCProfileIndication - out->push_back(sps[2]); // ...misc bits... -> profile_compatibility - out->push_back(sps[3]); // level_idc -> AVCLevelIndication + if (!sps.empty() && !pps.empty()) { + // Create the AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1. + // The beginning of the AVCDecoderConfiguration takes a few values from + // the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section + // defines the syntax in terms of RBSP, not NAL. The difference is the + // escaping of 00 00 01 and 00 00 02; see notes about + // "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4. + // It looks like 00 is not a valid value of profile_idc, so this distinction + // shouldn't be relevant here. And ffmpeg seems to ignore it. + out->push_back(1); // configurationVersion + out->push_back(sps[1]); // profile_idc -> AVCProfileIndication + out->push_back(sps[2]); // ...misc bits... -> profile_compatibility + out->push_back(sps[3]); // level_idc -> AVCLevelIndication - // Hardcode lengthSizeMinusOne to 3. This needs to match what ffmpeg uses - // when generating AVCParameterSamples (ISO/IEC 14496-15 section 5.3.2). - // There doesn't seem to be a clean way to get this from ffmpeg, but it's - // always 3. - out->push_back(static_cast(0xff)); + // Hardcode lengthSizeMinusOne to 3. This needs to match what ffmpeg uses + // when generating AVCParameterSamples (ISO/IEC 14496-15 section 5.3.2). + // There doesn't seem to be a clean way to get this from ffmpeg, but it's + // always 3. + out->push_back(static_cast(0xff)); - // Only support one SPS and PPS. - // ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably fine. - // This next byte is a reserved 0b111 + a 5-bit # of SPSs (1). - out->push_back(static_cast(0xe1)); - AppendU16(sps.size(), out); - out->append(sps.data(), sps.size()); - out->push_back(1); // # of PPSs. - AppendU16(pps.size(), out); - out->append(pps.data(), pps.size()); + // Only support one SPS and PPS. + // ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably + // fine. This next byte is a reserved 0b111 + a 5-bit # of SPSs (1). + out->push_back(static_cast(0xe1)); + AppendU16(sps.size(), out); + out->append(sps.data(), sps.size()); + out->push_back(1); // # of PPSs. + AppendU16(pps.size(), out); + out->append(pps.data(), pps.size()); - if (out->size() - avcc_len_pos != avcc_len) { - *error_message = - StrCat("internal error: anticipated AVCConfigurationBox length ", - avcc_len, ", but was actually ", out->size() - avcc_len_pos, - "; sps length ", sps.size(), ", pps length ", pps.size()); - return false; + if (out->size() - avcc_len_pos != avcc_len) { + *error_message = + StrCat("internal error: anticipated AVCConfigurationBox length ", + avcc_len, ", but was actually ", out->size() - avcc_len_pos, + "; sps length ", sps.size(), ", pps length ", pps.size()); + return false; + } + + } else { + out->append(extradata.data(), extradata.size()); } + if (out->size() - avc1_len_pos != avc1_len) { *error_message = StrCat("internal error: anticipated AVCSampleEntry length ", avc1_len, diff --git a/src/h264.h b/src/h264.h index a5ce0b0..d0a160a 100644 --- a/src/h264.h +++ b/src/h264.h @@ -31,15 +31,18 @@ // h264.h: H.264 decoding. For the most part, Moonfire NVR does not try to // understand the video codec. There's one exception. It must construct the // .mp4 sample description table, and for AVC, this includes the ISO/IEC -// 14496-15 section 5.2.4.1 AVCDecoderConfigurationRecord. ffmpeg supplies (as -// "extra data") an ISO/IEC 14496-10 Annex B byte stream containing SPS -// (sequence parameter set) and PPS (picture parameter set) NAL units from -// which this can be constructed. +// 14496-15 section 5.2.4.1 AVCDecoderConfigurationRecord. // -// ffmpeg of course also has logic for converting "extra data" to the -// AVCDecoderConfigurationRecord, but unfortunately it is not exposed except -// through ffmpeg's own generated .mp4 file. Extracting just this part of -// their .mp4 files would be more trouble than it's worth. +// When handling a RTSP input source, ffmpeg supplies as "extradata" an +// ISO/IEC 14496-10 Annex B byte stream containing SPS (sequence parameter +// set) and PPS (picture parameter set) NAL units from which this can be +// constructed. ffmpeg of course also has logic for converting "extradata" +// to the AVCDecoderConfigurationRecord, but unfortunately it is not exposed +// except through ffmpeg's own generated .mp4 file. Extracting just this part +// of their .mp4 files would be more trouble than it's worth. +// +// Just to make things interesting, when handling a .mp4 file, ffmpeg supplies +// as "extradata" an AVCDecoderConfiguration. #ifndef MOONFIRE_NVR_H264_H #define MOONFIRE_NVR_H264_H @@ -71,9 +74,9 @@ bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit, } // namespace // Gets a H.264 sample entry (AVCSampleEntry, which extends -// VisualSampleEntry), given the "extra_data", width, and height supplied by +// VisualSampleEntry), given the "extradata", width, and height supplied by // ffmpeg. -bool GetH264SampleEntry(re2::StringPiece extra_data, uint16_t width, +bool GetH264SampleEntry(re2::StringPiece extradata, uint16_t width, uint16_t height, std::string *out, std::string *error_message);