Skip to content

Commit

Permalink
feat: h266 support
Browse files Browse the repository at this point in the history
  • Loading branch information
@ireader
ireader committed Dec 10, 2022
1 parent 1e373ae commit 3239d05
Show file tree
Hide file tree
Showing 39 changed files with 1,700 additions and 40 deletions.
13 changes: 10 additions & 3 deletions libdash/test/dash-dynamic-test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "mpeg4-aac.h"
#include "mpeg4-avc.h"
#include "mpeg4-hevc.h"
#include "mpeg4-vvc.h"
#include "cstringext.h"
#include "sys/sync.hpp"
#include "sys/thread.h"
Expand Down Expand Up @@ -75,6 +76,7 @@ static int dash_live_onflv(void* param, int codec, const void* data, size_t byte
struct mpeg4_aac_t aac;
struct mpeg4_avc_t avc;
struct mpeg4_hevc_t hevc;
struct mpeg4_vvc_t vvc;
dash_playlist_t* dash = (dash_playlist_t*)param;

switch (codec)
Expand All @@ -86,9 +88,15 @@ static int dash_live_onflv(void* param, int codec, const void* data, size_t byte

case FLV_VIDEO_HVCC:
if (-1 == dash->adapation_video && mpeg4_hevc_decoder_configuration_record_load((const uint8_t*)data, bytes, &hevc) > 0)
dash->adapation_video = dash_mpd_add_video_adaptation_set(dash->mpd, dash->name.c_str(), MOV_OBJECT_HEVC, dash->width, dash->height, data, bytes);
dash->adapation_video = dash_mpd_add_video_adaptation_set(dash->mpd, dash->name.c_str(), MOV_OBJECT_H265, dash->width, dash->height, data, bytes);
break;

case FLV_VIDEO_VVCC:
if (-1 == dash->adapation_video && mpeg4_vvc_decoder_configuration_record_load((const uint8_t*)data, bytes, &vvc) > 0)
dash->adapation_video = dash_mpd_add_video_adaptation_set(dash->mpd, dash->name.c_str(), MOV_OBJECT_H266, dash->width, dash->height, data, bytes);
break;


case FLV_AUDIO_ASC:
if (-1 == dash->adapation_audio && mpeg4_aac_audio_specific_config_load((const uint8_t*)data, bytes, &aac) > 0)
{
Expand All @@ -101,9 +109,8 @@ static int dash_live_onflv(void* param, int codec, const void* data, size_t byte
return dash_mpd_input(dash->mpd, dash->adapation_audio, data, bytes, pts, dts, 0);

case FLV_VIDEO_H264:
return dash_mpd_input(dash->mpd, dash->adapation_video, data, bytes, pts, dts, flags ? MOV_AV_FLAG_KEYFREAME : 0);

case FLV_VIDEO_H265:
case FLV_VIDEO_H266:
return dash_mpd_input(dash->mpd, dash->adapation_video, data, bytes, pts, dts, flags ? MOV_AV_FLAG_KEYFREAME : 0);

default:
Expand Down
2 changes: 2 additions & 0 deletions libflv/include/flv-proto.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@
#define FLV_VIDEO_H264 7 // AVC
#define FLV_VIDEO_H265 12 // https://github.com/CDN-Union/H265
#define FLV_VIDEO_AV1 13 // https://aomediacodec.github.io/av1-isobmff
#define FLV_VIDEO_H266 14
#define FLV_VIDEO_AVCC 0x200 // AVCDecoderConfigurationRecord(ISO-14496-15)
#define FLV_VIDEO_HVCC 0x201 // HEVCDecoderConfigurationRecord(ISO-14496-15)
#define FLV_VIDEO_AV1C 0x202 // AV1CodecConfigurationRecord(av1-isobmff)
#define FLV_VIDEO_VVCC 0x203 // VVCDecoderConfigurationRecord(ISO-14496-15)

#define FLV_SCRIPT_METADATA 0x300 // onMetaData

Expand Down
4 changes: 4 additions & 0 deletions libflv/include/mpeg4-bits.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ static inline size_t mpeg4_bits_remain(struct mpeg4_bits_t* bits)
static inline void mpeg4_bits_skip(struct mpeg4_bits_t* bits, size_t n)
{
bits->bits += n;
if (bits->bits >= bits->size * 8)
{
bits->error = -1;
}
}

/// read 1-bit from bit stream(offset position)
Expand Down
73 changes: 73 additions & 0 deletions libflv/include/mpeg4-vvc.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#ifndef _mpeg_vvc_h
#define _mpeg_vvc_h

#include <stdint.h>
#include <stddef.h>

#if defined(__cplusplus)
extern "C" {
#endif

struct mpeg4_vvc_t
{
uint32_t lengthSizeMinusOne : 2; // 2bit,[0,3]
uint32_t ptl_present_flag : 1;

// valid on ptl_present_flag
uint32_t ols_idx : 9;
uint32_t num_sublayers : 3;
uint32_t constant_frame_rate : 2;
uint32_t chroma_format_idc : 2;
uint32_t bit_depth_minus8 : 2;
uint16_t max_picture_width;
uint16_t max_picture_height;
uint16_t avg_frame_rate;
struct
{
uint32_t num_bytes_constraint_info : 6;
uint32_t general_profile_idc : 7;
uint32_t general_tier_flag : 1;
uint32_t general_level_idc : 8;
uint32_t ptl_frame_only_constraint_flag : 1;
uint32_t ptl_multi_layer_enabled_flag : 1;
uint32_t ptl_sublayer_level_present_flag : 8;
uint8_t general_constraint_info[64];
uint8_t sublayer_level_idc[8 - 2];
uint8_t ptl_num_sub_profiles;
uint32_t *general_sub_profile_idc; // --> data
} native_ptl;

uint8_t numOfArrays;
struct
{
uint8_t array_completeness;
uint8_t type; // nalu type
uint16_t bytes;
uint8_t* data;
} nalu[64];

uint8_t array_completeness;
uint8_t data[4 * 1024];
size_t off;
};

int mpeg4_vvc_decoder_configuration_record_load(const uint8_t* data, size_t bytes, struct mpeg4_vvc_t* vvc);

int mpeg4_vvc_decoder_configuration_record_save(const struct mpeg4_vvc_t* vvc, uint8_t* data, size_t bytes);

int mpeg4_vvc_to_nalu(const struct mpeg4_vvc_t* vvc, uint8_t* data, size_t bytes);

int mpeg4_vvc_codecs(const struct mpeg4_vvc_t* vvc, char* codecs, size_t bytes);

int h266_annexbtomp4(struct mpeg4_vvc_t* vvc, const void* data, size_t bytes, void* out, size_t size, int* vcl, int* update);

int h266_mp4toannexb(const struct mpeg4_vvc_t* vvc, const void* data, size_t bytes, void* out, size_t size);

/// h266_is_new_access_unit H.266 new access unit(frame)
/// @return 1-new access, 0-not a new access
int h266_is_new_access_unit(const uint8_t* nalu, size_t bytes);

#if defined(__cplusplus)
}
#endif
#endif /* !_mpeg_vvc_h */
4 changes: 4 additions & 0 deletions libflv/libflv.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
<ClInclude Include="include\mpeg4-avc.h" />
<ClInclude Include="include\mpeg4-bits.h" />
<ClInclude Include="include\mpeg4-hevc.h" />
<ClInclude Include="include\mpeg4-vvc.h" />
<ClInclude Include="include\opus-head.h" />
<ClInclude Include="include\riff-acm.h" />
<ClInclude Include="include\webm-vpx.h" />
Expand All @@ -52,6 +53,7 @@
<ClCompile Include="source\hevc-annexbtomp4.c" />
<ClCompile Include="source\hevc-mp4toannexb.c" />
<ClCompile Include="source\mp3-header.c" />
<ClCompile Include="source\mpeg4-vvc.c" />
<ClCompile Include="source\mpeg4-aac-asc.c" />
<ClCompile Include="source\mpeg4-aac.c" />
<ClCompile Include="source\mpeg4-annexbtomp4.c" />
Expand All @@ -60,6 +62,8 @@
<ClCompile Include="source\mpeg4-mp4toannexb.c" />
<ClCompile Include="source\opus-head.c" />
<ClCompile Include="source\riff-acm.c" />
<ClCompile Include="source\vvc-annexbtomp4.c" />
<ClCompile Include="source\vvc-mp4toannexb.c" />
<ClCompile Include="source\webm-vpx.c" />
</ItemGroup>
<PropertyGroup Label="Globals">
Expand Down
12 changes: 12 additions & 0 deletions libflv/libflv.vcxproj.filters
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@
<ClInclude Include="include\riff-acm.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\mpeg4-vvc.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="source\flv-demuxer.c">
Expand Down Expand Up @@ -137,5 +140,14 @@
<ClCompile Include="source\riff-acm.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="source\mpeg4-vvc.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="source\vvc-annexbtomp4.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="source\vvc-mp4toannexb.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>
41 changes: 41 additions & 0 deletions libflv/source/flv-demuxer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "mpeg4-aac.h"
#include "mpeg4-avc.h"
#include "mpeg4-hevc.h"
#include "mpeg4-vvc.h"
#include "opus-head.h"
#include "aom-av1.h"
#include "amf0.h"
Expand All @@ -25,6 +26,7 @@ struct flv_demuxer_t
struct aom_av1_t av1;
struct mpeg4_avc_t avc;
struct mpeg4_hevc_t hevc;
struct mpeg4_vvc_t vvc;
} v;

flv_demuxer_handler handler;
Expand Down Expand Up @@ -222,6 +224,45 @@ static int flv_demuxer_video(struct flv_demuxer_t* flv, const uint8_t* data, int
return -EINVAL;
}
}
else if (FLV_VIDEO_H266 == video.codecid)
{
if (FLV_SEQUENCE_HEADER == video.avpacket)
{
// VVCDecoderConfigurationRecord
//assert(bytes > n + 5);
//mpeg4_vvc_decoder_configuration_record_load(data + n, bytes - n, &flv->v.vvc);
//return flv->handler(flv->param, FLV_VIDEO_VVCC, data + n, bytes - n, timestamp + video.cts, timestamp, 0);
}
else if (FLV_AVPACKET == video.avpacket)
{
// feat: h266_mp4toannexb support flv->v.vvc.numOfArrays == 0
//assert(flv->v.vvc.numOfArrays > 0); // parse VVCDecoderConfigurationRecord failed
//if (flv->v.vvc.numOfArrays > 0 && bytes > n) // 5 == bytes flv eof
{
// H.266
if (0 != flv_demuxer_check_and_alloc(flv, bytes + 4 * 1024))
return -ENOMEM;

n = h266_mp4toannexb(&flv->v.vvc, data + n, bytes - n, flv->ptr, flv->capacity);
if (n <= 0 || n > flv->capacity)
{
assert(0);
return -ENOMEM;
}
return flv->handler(flv->param, FLV_VIDEO_H266, flv->ptr, n, timestamp + video.cts, timestamp, (FLV_VIDEO_KEY_FRAME == video.keyframe) ? 1 : 0);
}
return -EINVAL;
}
else if (FLV_END_OF_SEQUENCE == video.avpacket)
{
return 0; // AVC end of sequence (lower level NALU sequence ender is not required or supported)
}
else
{
assert(0);
return -EINVAL;
}
}
else if (FLV_VIDEO_AV1 == video.codecid)
{
if (FLV_SEQUENCE_HEADER == video.avpacket)
Expand Down
4 changes: 2 additions & 2 deletions libflv/source/flv-header.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ int flv_video_tag_header_read(struct flv_video_tag_header_t* video, const uint8_
video->codecid = (buf[0] & 0x0F);
video->avpacket = FLV_AVPACKET;

if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_H266 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
{
if (len < 5)
return -1;
Expand Down Expand Up @@ -198,7 +198,7 @@ int flv_video_tag_header_write(const struct flv_video_tag_header_t* video, uint8

buf[0] = (video->keyframe << 4) /*FrameType*/ | (video->codecid & 0x0F) /*CodecID*/;

if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_H266 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
{
assert(FLV_SEQUENCE_HEADER == video->avpacket || FLV_AVPACKET == video->avpacket || FLV_END_OF_SEQUENCE == video->avpacket);
if (len < 5)
Expand Down
56 changes: 55 additions & 1 deletion libflv/source/flv-muxer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "aom-av1.h"
#include "mpeg4-aac.h"
#include "mpeg4-avc.h"
#include "mpeg4-vvc.h"
#include "mpeg4-hevc.h"
#include "mp3-header.h"
#include "opus-head.h"
Expand All @@ -35,7 +36,8 @@ struct flv_muxer_t
{
struct aom_av1_t av1;
struct mpeg4_avc_t avc;
struct mpeg4_hevc_t hevc;
struct mpeg4_hevc_t hevc;
struct mpeg4_vvc_t vvc;
} v;
int vcl; // 0-non vcl, 1-idr, 2-p/b
int update; // avc/hevc sequence header update
Expand Down Expand Up @@ -354,6 +356,58 @@ int flv_muxer_hevc(struct flv_muxer_t* flv, const void* data, size_t bytes, uint
return flv_muxer_h265(flv, pts, dts);
}

static int flv_muxer_h266(struct flv_muxer_t* flv, uint32_t pts, uint32_t dts)
{
int r;
int m;
struct flv_video_tag_header_t video;

video.codecid = FLV_VIDEO_H266;
if ( /*0 == flv->avc_sequence_header &&*/ flv->update && flv->v.vvc.numOfArrays >= 3) // vps + sps + pps
{
video.cts = 0;
video.keyframe = 1; // keyframe
video.avpacket = FLV_SEQUENCE_HEADER;
flv_video_tag_header_write(&video, flv->ptr + flv->bytes, flv->capacity - flv->bytes);
m = mpeg4_vvc_decoder_configuration_record_save(&flv->v.vvc, flv->ptr + flv->bytes + 5, flv->capacity - flv->bytes - 5);
if (m <= 0)
return -1; // invalid data

flv->video_sequence_header = 1; // once only
assert(flv->bytes + m + 5 <= flv->capacity);
r = flv->handler(flv->param, FLV_TYPE_VIDEO, flv->ptr + flv->bytes, m + 5, dts);
if (0 != r) return r;
}

// has video frame
if (flv->vcl && flv->video_sequence_header)
{
video.cts = pts - dts;
video.keyframe = 1 == flv->vcl ? FLV_VIDEO_KEY_FRAME : FLV_VIDEO_INTER_FRAME;
video.avpacket = FLV_AVPACKET;
flv_video_tag_header_write(&video, flv->ptr, flv->capacity);
assert(flv->bytes <= flv->capacity);
return flv->handler(flv->param, FLV_TYPE_VIDEO, flv->ptr, flv->bytes, dts);
}
return 0;
}

int flv_muxer_vvc(struct flv_muxer_t* flv, const void* data, size_t bytes, uint32_t pts, uint32_t dts)
{
if ((size_t)flv->capacity < bytes + sizeof(flv->v.vvc) /*HEVCDecoderConfigurationRecord*/)
{
if (0 != flv_muxer_alloc(flv, bytes + sizeof(flv->v.vvc)))
return -ENOMEM;
}

flv->bytes = 5;
flv->bytes += h266_annexbtomp4(&flv->v.vvc, data, bytes, flv->ptr + flv->bytes, flv->capacity - flv->bytes, &flv->vcl, &flv->update);
if (flv->bytes <= 5)
return -ENOMEM;

return flv_muxer_h266(flv, pts, dts);
}

int flv_muxer_av1(flv_muxer_t* flv, const void* data, size_t bytes, uint32_t pts, uint32_t dts)
{
int r;
Expand Down
6 changes: 3 additions & 3 deletions libflv/source/flv-parser.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#define FLV_HEADER_SIZE 9 // DataOffset included
#define FLV_TAG_HEADER_SIZE 11 // StreamID included

#define FLV_VIDEO_CODEC_NAME(codecid) (FLV_VIDEO_H264==(codecid) ? FLV_VIDEO_AVCC : (FLV_VIDEO_H265==(codecid) ? FLV_VIDEO_HVCC : FLV_VIDEO_AV1C))
#define FLV_VIDEO_CODEC_NAME(codecid) (FLV_VIDEO_H264==(codecid) ? FLV_VIDEO_AVCC : (FLV_VIDEO_H265==(codecid) ? FLV_VIDEO_HVCC : (FLV_VIDEO_H266==(codecid) ? FLV_VIDEO_VVCC : FLV_VIDEO_AV1C)))

static int flv_parser_audio(struct flv_audio_tag_header_t* audio, const uint8_t* data, size_t bytes, uint32_t timestamp, flv_parser_handler handler, void* param)
{
Expand All @@ -22,7 +22,7 @@ static int flv_parser_audio(struct flv_audio_tag_header_t* audio, const uint8_t*

static int flv_parser_video(struct flv_video_tag_header_t* video, const uint8_t* data, size_t bytes, uint32_t timestamp, flv_parser_handler handler, void* param)
{
if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
if (FLV_VIDEO_H264 == video->codecid || FLV_VIDEO_H265 == video->codecid || FLV_VIDEO_H266 == video->codecid || FLV_VIDEO_AV1 == video->codecid)
{
if (FLV_SEQUENCE_HEADER == video->avpacket)
{
Expand Down Expand Up @@ -181,7 +181,7 @@ int flv_parser_input(struct flv_parser_t* parser, const uint8_t* data, size_t by
parser->expect = 1;
n = flv_parser_append(parser, data, bytes, 1);
codec = (parser->ptr[0] & 0x0F);
if (FLV_VIDEO_H264 == codec || FLV_VIDEO_H265 == codec || FLV_VIDEO_AV1 == codec)
if (FLV_VIDEO_H264 == codec || FLV_VIDEO_H265 == codec || FLV_VIDEO_H266 == codec || FLV_VIDEO_AV1 == codec)
parser->expect = 5;
break;

Expand Down
4 changes: 3 additions & 1 deletion libflv/source/hevc-annexbtomp4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
#include <string.h>
#include <assert.h>

#define H265_NAL_BLA_W_LP 16
#define H265_NAL_RSV_IRAP 23
#define H265_NAL_VPS 32
#define H265_NAL_SPS 33
#define H265_NAL_PPS 34
Expand Down Expand Up @@ -374,7 +376,7 @@ static void hevc_handler(void* param, const uint8_t* nalu, size_t bytes)

// IRAP-1, B/P-2, other-0
if (mp4->vcl && nalutype < H265_NAL_VPS)
*mp4->vcl = 16<=nalutype && nalutype<=23 ? 1 : 2;
*mp4->vcl = H265_NAL_BLA_W_LP<=nalutype && nalutype<=H265_NAL_RSV_IRAP ? 1 : 2;

if (mp4->capacity >= mp4->bytes + bytes + 4)
{
Expand Down
Loading

0 comments on commit 3239d05

Please sign in to comment.