isomp4: CodecType and extra_data missing from avc1 track

[torokati44]

When running the following code:

use symphonia_core::{
    formats::{FormatOptions, FormatReader},
    io::MediaSourceStream,
};
use symphonia_format_isomp4::IsoMp4Reader;

fn main() {
    let stream = std::fs::File::open("BigBuckBunny_1s.mp4").unwrap();
    let source = MediaSourceStream::new(Box::new(stream), Default::default());
    let reader = IsoMp4Reader::try_new(source, &FormatOptions::default()).unwrap();
    dbg!(reader.tracks());
}

I get this as (part of) the output:

    Track {
        id: 0,
        codec_params: CodecParameters {
            codec: CodecType(
                0,
            ),
            sample_rate: None,
            time_base: Some(
                TimeBase {
                    numer: 1,
                    denom: 90000,
                },
            ),
            n_frames: Some(
                105000,
            ),
            start_ts: 0,
            sample_format: None,
            bits_per_sample: None,
            bits_per_coded_sample: None,
            channels: None,
            channel_layout: None,
            delay: None,
            padding: None,
            max_frames_per_packet: None,
            packet_data_integrity: false,
            verification_check: None,
            frames_per_block: None,
            extra_data: None,
        },
        language: None,
    },

mp4box.js correctly identifies the codec as avc1.42421f, and the file is also valid otherwise.

The missing extra_data may not be an actual bug, as H.264 has two bitstream formats ("Annex B", and "AVCC"), and in one of them (albeit, the one geared more towards streaming, not storing in files), the extradata is in-band, transmitted as regular data packets (or, samples, or call it whatever you want).

The file for testing:
https://github.com/pdeljanov/Symphonia/assets/288816/a92343a0-46f4-411e-9c72-a2328e70d78a

[pdeljanov]

This isn't really a bug so much as support for non-audio tracks is very weak right now. A big reason for this is that the current interfaces are very audio specific (e.g., Packet making no distinction between decode and presentation timestamps, CodecParameters being audio only, etc.). I plan to improve this in the dev-0.6 branch which I've just started pushing changes to.

While I don't think I'll be writing any video decoders soon, a project goal is to allow Symphonia demuxers to work with wrapped third-party video decoders like libvpx, etc.

[sscobici]

Fixed in dev-0.6 branch

    Track {
        id: 1,
        codec_params: Some(
            Video(
                VideoCodecParameters {
                    codec: VideoCodecId(
                        2820,
                    ),
                    profile: Some(
                        CodecProfile(
                            77,
                        ),
                    ),
                    level: Some(
                        40,
                    ),
                    width: Some(
                        320,
                    ),
                    height: Some(
                        240,
                    ),
                    extra_data: [
                        VideoExtraData {
                            id: VideoExtraDataId(
                                1,
                            ),
                            data: [
                                1,
                                77,
                                64,
                                40,
                                255,
                                225,
                                0,
                                27,
                                103,
                                77,
                                64,
                                40,
                                236,
                                160,
                                160,
                                253,
                                135,
                                53,
                                1,
                                1,
                                1,
                                64,
                                0,
                                0,
                                3,
                                0,
                                64,
                                0,
                                175,
                                200,
                                3,
                                198,
                                12,
                                101,
                                128,
                                1,
                                0,
                                5,
                                104,
                                234,
                                225,
                                50,
                                200,
                            ],
                        },
                    ],
                },
            ),
        ),
        language: None,
        time_base: Some(
            TimeBase {
                numer: 1,
                denom: 90000,
            },
        ),
        num_frames: Some(
            105000,
        ),
        start_ts: 0,
        delay: None,
        padding: None,
        flags: TrackFlags(
            0x0,
        ),
    },
    Track {
        id: 2,
        codec_params: Some(
            Audio(
                AudioCodecParameters {
                    codec: AudioCodecId(
                        4103,
                    ),
                    profile: None,
                    sample_rate: Some(
                        44100,
                    ),
                    sample_format: None,
                    bits_per_sample: None,
                    bits_per_coded_sample: None,
                    channels: None,
                    max_frames_per_packet: None,
                    verification_check: None,
                    frames_per_block: None,
                    extra_data: Some(
                        [
                            18,
                            16,
                            86,
                            229,
                            0,
                        ],
                    ),
                },
            ),
        ),
        language: None,
        time_base: Some(
            TimeBase {
                numer: 1,
                denom: 44100,
            },
        ),
        num_frames: Some(
            46080,
        ),
        start_ts: 0,
        delay: None,
        padding: None,
        flags: TrackFlags(
            0x0,
        ),
    },
]```