一：整体代码函数预览
static ngx_int_t
ngx_rtmp_mp4_postconfiguration(ngx_conf_t *cf)
{ngx_rtmp_play_main_conf_t *pmcf;ngx_rtmp_play_fmt_t **pfmt, *fmt;pmcf = ngx_rtmp_conf_get_module_main_conf(cf, ngx_rtmp_play_module);pfmt = ngx_array_push(&pmcf->fmts);if (pfmt == NULL) {return NGX_ERROR;}fmt = ngx_pcalloc(cf->pool, sizeof(ngx_rtmp_play_fmt_t));if (fmt == NULL) {return NGX_ERROR;}*pfmt = fmt;ngx_str_set(&fmt->name, "mp4-format");ngx_str_set(&fmt->pfx, "mp4:");ngx_str_set(&fmt->sfx, ".mp4");fmt->init = ngx_rtmp_mp4_init; // 初始化fmt->done = ngx_rtmp_mp4_done; // 完成fmt->seek = ngx_rtmp_mp4_seek; // seekfmt->start = ngx_rtmp_mp4_start; // 开始fmt->stop = ngx_rtmp_mp4_stop; // 结束fmt->send = ngx_rtmp_mp4_send; // 发送数据return NGX_OK;
}二：数据结构定义
| 定义 | 标准定义 | nginx 数据结构定义 |
| stsc 【Sample To Chunk Box】 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t first_chunk; uint32_t samples_per_chunk; uint32_t sample_descrption_index; } ngx_rtmp_mp4_chunk_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_chunk_entry_t entries[0]; } ngx_rtmp_mp4_chunks_t; |
| stts 【Decoding Time to Sample Box】 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t sample_count; uint32_t sample_delta; } ngx_rtmp_mp4_time_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_time_entry_t entries[0]; } ngx_rtmp_mp4_times_t; |
| ctts 【Composition Time to Sample Box】 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t sample_count; uint32_t sample_offset; } ngx_rtmp_mp4_delay_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_delay_entry_t entries[0]; } ngx_rtmp_mp4_delays_t; |
| stss 【Sync Sample Box】 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; uint32_t entry_count; uint32_t entries[0]; } ngx_rtmp_mp4_keys_t; |
| stsz 【Sample Size Boxes】 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; 4 字节 uint32_t sample_size; uint32_t sample_count; uint32_t entries[0]; } ngx_rtmp_mp4_sizes_t; |
| stco 【Chunk Offset Box】 co64 |  | type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; 4 字节固定 uint32_t entry_count; uint32_t entries[0]; } ngx_rtmp_mp4_offsets_t; typedef struct { uint32_t version_flags; uint32_t entry_count; uint64_t entries[0]; } ngx_rtmp_mp4_offsets64_t; |
三：数据结构解析
static ngx_rtmp_mp4_box_t ngx_rtmp_mp4_boxes[] = {{ ngx_rtmp_mp4_make_tag('t','r','a','k'), ngx_rtmp_mp4_parse_trak },{ ngx_rtmp_mp4_make_tag('m','d','i','a'), ngx_rtmp_mp4_parse },{ ngx_rtmp_mp4_make_tag('m','d','h','d'), ngx_rtmp_mp4_parse_mdhd },{ ngx_rtmp_mp4_make_tag('h','d','l','r'), ngx_rtmp_mp4_parse_hdlr },{ ngx_rtmp_mp4_make_tag('m','i','n','f'), ngx_rtmp_mp4_parse },{ ngx_rtmp_mp4_make_tag('s','t','b','l'), ngx_rtmp_mp4_parse },{ ngx_rtmp_mp4_make_tag('s','t','s','d'), ngx_rtmp_mp4_parse_stsd },{ ngx_rtmp_mp4_make_tag('s','t','s','c'), ngx_rtmp_mp4_parse_stsc }, // 记录了每个chunk中包含多少sample{ ngx_rtmp_mp4_make_tag('s','t','t','s'), ngx_rtmp_mp4_parse_stts }, // sample 解码时间的压缩表{ ngx_rtmp_mp4_make_tag('c','t','t','s'), ngx_rtmp_mp4_parse_ctts }, // 帧解码到渲染的时间差值，通常用在B帧的场景，sample的CTS与DTS的时间差的压缩表{ ngx_rtmp_mp4_make_tag('s','t','s','s'), ngx_rtmp_mp4_parse_stss }, // 关键帧映射表{ ngx_rtmp_mp4_make_tag('s','t','s','z'), ngx_rtmp_mp4_parse_stsz }, // 每帧数据的大小{ ngx_rtmp_mp4_make_tag('s','t','z','2'), ngx_rtmp_mp4_parse_stz2 },{ ngx_rtmp_mp4_make_tag('s','t','c','o'), ngx_rtmp_mp4_parse_stco }, // 记录了chunk对应的offset{ ngx_rtmp_mp4_make_tag('c','o','6','4'), ngx_rtmp_mp4_parse_co64 },{ ngx_rtmp_mp4_make_tag('a','v','c','1'), ngx_rtmp_mp4_parse_avc1 },{ ngx_rtmp_mp4_make_tag('a','v','c','C'), ngx_rtmp_mp4_parse_avcC }, // sps pps 解析{ ngx_rtmp_mp4_make_tag('m','p','4','a'), ngx_rtmp_mp4_parse_mp4a },{ ngx_rtmp_mp4_make_tag('m','p','4','v'), ngx_rtmp_mp4_parse_mp4v },{ ngx_rtmp_mp4_make_tag('e','s','d','s'), ngx_rtmp_mp4_parse_esds },{ ngx_rtmp_mp4_make_tag('.','m','p','3'), ngx_rtmp_mp4_parse_mp3 },{ ngx_rtmp_mp4_make_tag('n','m','o','s'), ngx_rtmp_mp4_parse_nmos },{ ngx_rtmp_mp4_make_tag('s','p','e','x'), ngx_rtmp_mp4_parse_spex },{ ngx_rtmp_mp4_make_tag('w','a','v','e'), ngx_rtmp_mp4_parse }
};ngx_rtmp_mp4_parse_stsc 记录了每个chunk中包含多少sample
上图数据解析
00 00 00 28(size) --> 40 字节总大小包含的自己在内的所有数据：
00 00 00 28(size) 73 74 73 63（stsc） 00 00 00 00(version_flags) 00 00 00 02(entry_count) 00 00 00 01(first_chunk) 00 00 00 02(samples_per_chunk) 00 00 00 01(sample_descrption_index) 00 00 00 02(first_chunk) 00 00 00 01(samples_per_chunk) 00 00 00 01(sample_descrption_index)
static ngx_int_t
ngx_rtmp_mp4_parse_stsc(ngx_rtmp_session_t *s, u_char *pos, u_char *last)
{ngx_rtmp_mp4_ctx_t *ctx;ngx_rtmp_mp4_track_t *t;ctx = ngx_rtmp_get_module_ctx(s, ngx_rtmp_mp4_module);t = ctx->track;if (t == NULL) {return NGX_OK;}t->chunks = (ngx_rtmp_mp4_chunks_t *) pos; // 内存数据上面的解析if (pos + sizeof(*t->chunks) + ngx_rtmp_r32(t->chunks->entry_count) *sizeof(t->chunks->entries[0])<= last)// 内存越界判断{return NGX_OK;}t->chunks = NULL;return NGX_ERROR;
}ngx_rtmp_mp4_parse_stco 记录了chunk对应的offset
00 00 03 D4（980 长度） 73 74 63 6F （stco）00 00 00 00(version_flags) 00 00 00 F1( entry_count 241 ) 00 00 00 30( entrie) 00 00 10 4D ( entrie)00 00 20 1C( entrie) 00 00 29 38( entrie) 00 00 49 2D( entrie) 00 00 70 C9( entrie) 00 00 B1 C1 ( entrie) ...............
static ngx_int_t
ngx_rtmp_mp4_parse_stco(ngx_rtmp_session_t *s, u_char *pos, u_char *last)
{ngx_rtmp_mp4_ctx_t *ctx;ngx_rtmp_mp4_track_t *t;ctx = ngx_rtmp_get_module_ctx(s, ngx_rtmp_mp4_module);t = ctx->track;if (t == NULL) {return NGX_OK;}t->offsets = (ngx_rtmp_mp4_offsets_t *) pos; // 数据结构的赋值if (pos + sizeof(*t->offsets) + ngx_rtmp_r32(t->offsets->entry_count) *sizeof(t->offsets->entries[0])<= last) // 内存越界判断{return NGX_OK;}t->offsets = NULL;return NGX_ERROR;
}其他数据结构的解析与上面的流程基本一样
四：数据处理流程
五：play 流程代码分析
ffplay -analyzeduration 1000000 "rtmp://10.90.103.5/vod/0001.mp4
1> ngx_rtmp_mp4_init 初始化变量解析 mp4 moov box 的数据结构
2> ngx_rtmp_mp4_seek (ngx_rtmp_session_t *s, ngx_file_t *f, ngx_uint_t timestamp)， timestamp = 0 ，seek 时间戳为零标识从头开始播放。ngx_rtmp_mp4_seek_track 有两个轨道，音频轨和视频轨，分别进行seek
static ngx_int_t
ngx_rtmp_mp4_seek_track(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t,ngx_int_t timestamp)
{ngx_rtmp_mp4_cursor_t *cr;cr = &t->cursor;ngx_memzero(cr, sizeof(*cr));if (ngx_rtmp_mp4_seek_time(s, t, ngx_rtmp_mp4_from_rtmp_timestamp( // ngx_rtmp_mp4_from_rtmp_timestamp 时间戳的t, timestamp)) != NGX_OK ||ngx_rtmp_mp4_seek_key(s, t) != NGX_OK ||ngx_rtmp_mp4_seek_chunk(s, t) != NGX_OK ||ngx_rtmp_mp4_seek_size(s, t) != NGX_OK ||ngx_rtmp_mp4_seek_delay(s, t) != NGX_OK){return NGX_ERROR;}cr->valid = 1;return NGX_OK;
} // 时间戳的转换函数，mp4 文件到rtmp
// rtmp_mp4 存放的是timescale 时间戳 通常是 12288
// rtmp 时间刻度通常是 1000
static ngx_inline uint32_t
ngx_rtmp_mp4_to_rtmp_timestamp(ngx_rtmp_mp4_track_t *t, uint64_t ts) // 给定一个时间值 从rtmp_mp4 12288 刻度转换到 1000
{return (uint32_t) (ts * 1000 / t->time_scale);
}
static ngx_inline uint32_t
ngx_rtmp_mp4_from_rtmp_timestamp(ngx_rtmp_mp4_track_t *t, uint32_t ts) // 给定一个时间值 从rtmp 1000 刻度转换到12288
{return (uint64_t) ts * t->time_scale / 1000;
}ngx_rtmp_mp4_seek_time :
| stts： typedef struct { uint32_t sample_count; uint32_t sample_delta; } ngx_rtmp_mp4_time_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_time_entry_t entries[0]; } ngx_rtmp_mp4_times_t; |  |  |
static ngx_int_t
ngx_rtmp_mp4_seek_time(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t,uint32_t timestamp)
{ngx_rtmp_mp4_cursor_t *cr;ngx_rtmp_mp4_time_entry_t *te;uint32_t dt;if (t->times == NULL) {return NGX_ERROR;}cr = &t->cursor; // 对应音频或者视频轨道的游标器te = t->times->entries; // times --> ngx_rtmp_mp4_times_t// sample_counts 6,1,5,1,5,1,1,1,1,1// sample_deltas 1024,3040,1024,3088,1024,1008,1024,1032,1024,1048 // 整个循环就是处理，遍历sample_counts个sample_deltas值与 参数 timestamp比大小. 1. 小于记录 cr->timestamp += dt,while (cr->time_pos < ngx_rtmp_r32(t->times->entry_count)) {dt = ngx_rtmp_r32(te->sample_delta) * ngx_rtmp_r32(te->sample_count); // 242 * 1024 = 22757376 // cr->timestamp + dt >= timestamp ，在这个范围说明找到if (cr->timestamp + dt >= timestamp) {if (te->sample_delta == 0) {return NGX_ERROR;}cr->time_count = (timestamp - cr->timestamp) /ngx_rtmp_r32(te->sample_delta); // sample_delta 相同，cr->time_count 等于当前时间戳距离这个entry开始时间错的插值，的第几个deltacr->timestamp += ngx_rtmp_r32(te->sample_delta) * cr->time_count; // seek 的时间戳，就是把 cr->timestamp 修改当前的时间戳cr->pos += cr->time_count; // 记录当前seek 对应时间戳具体的位置break;}// 没找到，cr->timestamp += dt; // 记录值进行累加，访问下一个cr->pos += ngx_rtmp_r32(te->sample_count); // 记录pos 的位置cr->time_pos++; // 记录已经访问第几个entryte++; // te++ 下一个entry}if (cr->time_pos >= ngx_rtmp_r32(t->times->entry_count)) {return NGX_ERROR;}return NGX_OK;
}ngx_rtmp_mp4_seek_key
| stss typedef struct { uint32_t version_flags; uint32_t entry_count; // 个数 uint32_t entries[0]; // pos } ngx_rtmp_mp4_keys_t; |  |
static ngx_int_t
ngx_rtmp_mp4_seek_key(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{ngx_rtmp_mp4_cursor_t *cr;uint32_t *ke;ngx_int_t dpos;cr = &t->cursor; // 对应音频或者视频轨道的游标器if (t->keys == NULL) {return NGX_OK;}// cr->key_pos 初始值为0 ，t->keys->entry_count 关键帧的个数while (cr->key_pos < ngx_rtmp_r32(t->keys->entry_count)) {// 遍历每一个entry得到帧号，与当前pos位置进行对比，找到当前位置的下一个关键帧if (ngx_rtmp_r32(t->keys->entries[cr->key_pos]) > cr->pos) { // cr->pos 上面seek 后具体位置break;}cr->key_pos++;}if (cr->key_pos >= ngx_rtmp_r32(t->keys->entry_count)) {return NGX_OK;}ke = &t->keys->entries[cr->key_pos]; // dpos = ngx_rtmp_r32(*ke) - cr->pos - 1; // 计算距离下一个关键帧的差值cr->key = 1;/* TODO: range version needed */for (; dpos > 0; --dpos) {ngx_rtmp_mp4_next_time(s, t);}return NGX_OK;
}ngx_rtmp_mp4_seek_chunk
| stsc typedef struct { uint32_t first_chunk; uint32_t samples_per_chunk; uint32_t sample_descrption_index; } ngx_rtmp_mp4_chunk_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_chunk_entry_t entries[0]; } ngx_rtmp_mp4_chunks_t; |  |  |
static ngx_int_t
ngx_rtmp_mp4_seek_chunk(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{ngx_rtmp_mp4_cursor_t *cr;ngx_rtmp_mp4_chunk_entry_t *ce, *nce;ngx_uint_t pos, dpos, dchunk;cr = &t->cursor; // 对应音频或者视频轨道的游标器if (t->chunks == NULL || t->chunks->entry_count == 0) {cr->chunk = 1;return NGX_OK;}ce = t->chunks->entries; // entry 实体结构 pos = 0;// cr->chunk_pos 初始值是0 while (cr->chunk_pos + 1 < ngx_rtmp_r32(t->chunks->entry_count)) {nce = ce + 1; // 下一个entrydpos = (ngx_rtmp_r32(nce->first_chunk) -ngx_rtmp_r32(ce->first_chunk)) *ngx_rtmp_r32(ce->samples_per_chunk); // 距离下一个chunk总共有多少点，（2-1） * 2 = 2 帧if (pos + dpos > cr->pos) { // 判断当前位置是不是在这个chunk 中break;}pos += dpos; // pos 从头遍历累加 ce++; // 访问下一个entrycr->chunk_pos++; // chunk_pos ++，记录游标当前在第几个chunk位置 }if (ce->samples_per_chunk == 0) {return NGX_ERROR;}dchunk = (cr->pos - pos) / ngx_rtmp_r32(ce->samples_per_chunk); // seek 之后的当前pos 位置，与找到chunk的pos差值，相同连续的chunk 可能不记录写进去。// 比如: first_chunk 1,2,3,4,5,6,12,13// samples_per_chunk 1,4,2,4,2,4,3, 4 ---> 1 + 4 + 2 + 4 + 2 + 4 = 17 , 18 --> 38 之间都是; 这个时候会有一个 dchunk差值cr->chunk = ngx_rtmp_r32(ce->first_chunk) + dchunk; // 得到当前游标chunk的值，真实对应chunk 位置。cr->chunk_pos = (ngx_uint_t) (ce - t->chunks->entries); // cr->chunk_pos 记录的是存储的真实解析位置cr->chunk_count = (ngx_uint_t) (cr->pos - pos - dchunk *ngx_rtmp_r32(ce->samples_per_chunk)); // 剩余是 chunk 偏移的变量帧return ngx_rtmp_mp4_update_offset(s, t);
}static ngx_int_t
ngx_rtmp_mp4_update_offset(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{ngx_rtmp_mp4_cursor_t *cr;ngx_uint_t chunk;cr = &t->cursor;if (cr->chunk < 1) {return NGX_ERROR;}chunk = cr->chunk - 1; // 数组访问 -1 if (t->offsets) {if (chunk >= ngx_rtmp_r32(t->offsets->entry_count)) {return NGX_ERROR;}cr->offset = (off_t) ngx_rtmp_r32(t->offsets->entries[chunk]); // 取出第几个chunk偏移大小进行赋值cr->size = 0;return NGX_OK;}if (t->offsets64) {if (chunk >= ngx_rtmp_r32(t->offsets64->entry_count)) {return NGX_ERROR;}cr->offset = (off_t) ngx_rtmp_r64(t->offsets64->entries[chunk]);cr->size = 0;return NGX_OK;}return NGX_ERROR;
}ngx_rtmp_mp4_seek_size
| stsz typedef struct { uint32_t version_flags; 4 字节 uint32_t sample_size; uint32_t sample_count; uint32_t entries[0]; } ngx_rtmp_mp4_sizes_t; |  |
static ngx_int_t
ngx_rtmp_mp4_seek_size(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{ngx_rtmp_mp4_cursor_t *cr;ngx_uint_t pos;cr = &t->cursor; // 当前游标if (cr->chunk_count > cr->pos) { // 》 ？？return NGX_ERROR;}if (t->sizes) {if (t->sizes->sample_size) {cr->size = ngx_rtmp_r32(t->sizes->sample_size);cr->offset += cr->size * cr->chunk_count;return NGX_OK;}if (cr->pos >= ngx_rtmp_r32(t->sizes->sample_count)) {return NGX_ERROR;}for (pos = 1; pos <= cr->chunk_count; ++pos) { // chunk_count ??cr->offset += ngx_rtmp_r32(t->sizes->entries[cr->pos - pos]); }cr->size_pos = cr->pos; // seek 时间戳当前poscr->size = ngx_rtmp_r32(t->sizes->entries[cr->size_pos]); // 获取当前pos对应帧数据大小return NGX_OK;}if (t->sizes2) {if (cr->size_pos >= ngx_rtmp_r32(t->sizes2->sample_count)) {return NGX_ERROR;}cr->size_pos = cr->pos;return NGX_OK;}return NGX_ERROR;
}ngx_rtmp_mp4_seek_delay
| ctts typedef struct { uint32_t sample_count; uint32_t sample_offset; } ngx_rtmp_mp4_delay_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_delay_entry_t entries[0]; } ngx_rtmp_mp4_delays_t; |  |
static ngx_int_t
ngx_rtmp_mp4_seek_delay(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{ngx_rtmp_mp4_cursor_t *cr;ngx_rtmp_mp4_delay_entry_t *de;uint32_t pos, dpos;cr = &t->cursor;if (t->delays == NULL) {return NGX_OK;}pos = 0;de = t->delays->entries;while (cr->delay_pos < ngx_rtmp_r32(t->delays->entry_count)) { // 遍历所有entrydpos = ngx_rtmp_r32(de->sample_count); // 2 表示：sample的个数if (pos + dpos > cr->pos) { // 0 + 2 > 0 , 当前pos 帧在其中cr->delay_count = cr->pos - pos; // 差值 countcr->delay = ngx_rtmp_r32(de->sample_offset); // 偏移大小break;}cr->delay_pos++; // 记录entry访问的位置pos += dpos; // pos 累加计数de++; // 下一个entry}if (cr->delay_pos >= ngx_rtmp_r32(t->delays->entry_count)) {return NGX_OK;}return NGX_OK;
}3> ngx_rtmp_mp4_next play 播放一直取下一帧数据
static ngx_int_t
ngx_rtmp_mp4_next(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{if (ngx_rtmp_mp4_next_time(s, t) != NGX_OK ||ngx_rtmp_mp4_next_key(s, t) != NGX_OK ||ngx_rtmp_mp4_next_chunk(s, t) != NGX_OK ||ngx_rtmp_mp4_next_size(s, t) != NGX_OK ||ngx_rtmp_mp4_next_delay(s, t) != NGX_OK){t->cursor.valid = 0;return NGX_ERROR;}t->cursor.valid = 1;return NGX_OK;
}六：代码调试
使用进行查看变量的值： ngx_rtmp_r32(t->delays->entry_count)
