Overview

Implementing dynamic video stream processing requires capturing individual frames. This approach enables real-time manipulation of video content but may introduce CPU overhead during decoding and image processing, potentially causing playback stuttering (performance optimization discussed later).

LibVLCSharp Callback Implementation

To capture video frames in WinForms using LibVLCSharp, configure two essential callback methods: SetVideoFormatCallbacks and SetVideoCallbacks. These methods establish the framework for frame extraction and processing.

Method Definitions

/// <summary>
/// Configures video format specifications and dimensions.
/// Must be used with libvlc_video_set_callbacks().
/// </summary>
/// <param name="formatDelegate">Callback for video format selection (required)</param>
/// <param name="cleanupDelegate">Callback for resource deallocation (optional)</param>
public void SetVideoFormatCallbacks(
  MediaPlayer.LibVLCVideoFormatCb formatDelegate,
  MediaPlayer.LibVLCVideoCleanupCb? cleanupDelegate)
{
  this._videoFormatDelegate = formatDelegate ?? throw new ArgumentNullException(nameof(formatDelegate));
  this._videoCleanupDelegate = cleanupDelegate;
  MediaPlayer.Native.LibVLCVideoSetFormatCallbacks(this.NativeReference, 
    MediaPlayer.VideoFormatCallbackHandle, 
    cleanupDelegate == null ? null : this._videoCleanupDelegate);
}

/// <summary>
/// Establishes custom memory buffer rendering for decoded video.
/// Requires prior format configuration via libvlc_video_set_format() or 
/// libvlc_video_set_format_callbacks().
/// Note: Custom memory rendering is less efficient than window embedding.
/// </summary>
/// <param name="lockDelegate">Callback for video memory locking (required)</param>
/// <param name="unlockDelegate">Callback for memory unlocking (optional)</param>
/// <param name="displayDelegate">Callback for video display (optional)</param>
public void SetVideoCallbacks(
  MediaPlayer.LibVLCVideoLockCb lockDelegate,
  MediaPlayer.LibVLCVideoUnlockCb? unlockDelegate,
  MediaPlayer.LibVLCVideoDisplayCb? displayDelegate)
{
  this._videoLockDelegate = lockDelegate ?? throw new ArgumentNullException(nameof(lockDelegate));
  this._videoUnlockDelegate = unlockDelegate;
  this._videoDisplayDelegate = displayDelegate;
  MediaPlayer.Native.LibVLCVideoSetCallbacks(this.NativeReference, 
    MediaPlayer.VideoLockCallbackHandle, 
    unlockDelegate == null ? null : MediaPlayer.VideoUnlockCallbackHandle, 
    displayDelegate == null ? null : MediaPlayer.VideoDisplayCallbackHandle, 
    GCHandle.ToIntPtr(this._gcHandle));
}

Callback Mechanism

• SetVideoFormatCallbacks: formatDelegate initializes image data, cleanupDelegate manages resource cleanup
• SetVideoCallbacks: lockDelegate handles frame decoding, unlockDelegate releases buffer, displayDelegate renders output

Frame Data Acquisition

In WinForms applications, captured video frames require transfer to an image buffer. Implement this using memory-mapped files combined with Bitmap objects. Within the lockDelegate, create necessary memory structures based on video parameters:

var bufferSize = stride * height;
_memoryFile = MemoryMappedFile.CreateNew(null, bufferSize);
_memoryAccessor = _memoryFile.CreateViewAccessor();
var bufferHandle = _memoryAccessor.SafeMemoryMappedViewHandle.DangerousGetHandle();
userData = bufferHandle;

var frameParams = new { width = width, height = height };
_context.Post(state => {
  _frameBitmap = new Bitmap((int)frameParams.width, (int)frameParams.height, 
    PixelFormat.Format32bppRgb);
  _pixelBuffer = new byte[(int)frameParams.width * (int)frameParams.height * 4];
  _bitmapContext = Graphics.FromImage(_frameBitmap);
}, null);

During frame decoding, write video data to the memory-mapped file:

Marshal.WriteIntPtr(planePointers, userData);

Frame Display Process

The displayDelegate callback renders the image. Copy data from the memory-mapped file to the Bitmap, trigger external rendering events for processing, and notify completion to update the display:

try {
  var bitmapData = _frameBitmap.LockBits(
    new Rectangle(0, 0, _videoWidth, _videoHeight),
    ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb);
  
  _memoryAccessor.ReadArray(8, _pixelBuffer, 0, _pixelBuffer.Length);
  Marshal.Copy(_pixelBuffer, 0, bitmapData.Scan0, _pixelBuffer.Length);
  _frameBitmap.UnlockBits(bitmapData);

  FrameReady.Invoke(_bitmapContext);
}
catch (Exception ex) {
  Console.WriteLine(ex.Message);
}

Task.Run(() => {
  var frameCopy = _frameBitmap.Clone(
    new Rectangle(0, 0, _frameBitmap.Width, _frameBitmap.Height),
    PixelFormat.Format32bppRgb);

  _context.Post(state => {
    FrameUpdated.Invoke(frameCopy);
  }, null);
});

Implementation Example

Custom Video Provider Class

internal class VideoFrameProvider
{
    public event Action<Graphics> FrameReady = (_) => { };
    public event Action<Bitmap> FrameUpdated = (_) => { };

    private MemoryMappedFile _memoryFile;
    private MemoryMappedViewAccessor _memoryAccessor;
    private int _videoWidth;
    private int _videoHeight;
    private Bitmap _frameBitmap;
    private Graphics _bitmapContext;
    private byte[] _pixelBuffer;
    private readonly MediaPlayer _player;
    private readonly SynchronizationContext _context;

    public VideoFrameProvider(MediaPlayer player, SynchronizationContext context)
    {
        _player = player;
        _player.SetVideoFormatCallbacks(SetupVideoFormat, CleanupVideo);
        _player.SetVideoCallbacks(LockFrameBuffer, null, RenderFrame);
        _context = context;
    }

    private uint SetupVideoFormat(ref IntPtr userData, IntPtr chroma, 
        ref uint width, ref uint height, ref uint stride, ref uint scanlines)
    {
        FourCCConverter.ToFourCC("RV32", chroma);

        var media = _player.Media;
        if (media != null)
        {
            foreach (MediaTrack track in media.Tracks)
            {
                if (track.TrackType == TrackType.Video)
                {
                    var videoInfo = track.Data.Video;
                    if (videoInfo.Width > 0 && videoInfo.Height > 0)
                    {
                        width = videoInfo.Width;
                        height = videoInfo.Height;
                        if (videoInfo.SarDen != 0)
                        {
                            width = width * videoInfo.SarNum / videoInfo.SarDen;
                        }
                    }
                    break;
                }
            }
        }

        stride = AlignToBoundary((width * 32) / 8, 32);
        scanlines = AlignToBoundary(height, 32);

        _videoWidth = (int)width;
        _videoHeight = (int)height;

        var bufferSize = stride * scanlines;
        _memoryFile = MemoryMappedFile.CreateNew(null, bufferSize);

        _context.Post(state => {
            _frameBitmap = new Bitmap((int)width, (int)height, PixelFormat.Format32bppRgb);
            _pixelBuffer = new byte[(int)width * (int)height * 4];
            _bitmapContext = Graphics.FromImage(_frameBitmap);
        }, null);

        _memoryAccessor = _memoryFile.CreateViewAccessor();
        var handle = _memoryAccessor.SafeMemoryMappedViewHandle.DangerousGetHandle();
        userData = handle;
        return 1;
    }

    private void CleanupVideo(ref IntPtr userData)
    {
        _context.Post(state => ReleaseResources(), null);
    }

    private IntPtr LockFrameBuffer(IntPtr userData, IntPtr planePointers)
    {
        Marshal.WriteIntPtr(planePointers, userData);
        return userData;
    }

    private void RenderFrame(IntPtr userData, IntPtr picture)
    {
        if (_frameBitmap == null) return;

        try
        {
            var bitmapData = _frameBitmap.LockBits(
                new Rectangle(0, 0, _videoWidth, _videoHeight),
                ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb);
            
            _memoryAccessor.ReadArray(8, _pixelBuffer, 0, _pixelBuffer.Length);
            Marshal.Copy(_pixelBuffer, 0, bitmapData.Scan0, _pixelBuffer.Length);
            _frameBitmap.UnlockBits(bitmapData);

            FrameReady.Invoke(_bitmapContext);
        }
        catch (Exception ex)
        {
            Console.WriteLine(ex.Message);
        }

        Task.Run(() => {
            var frameCopy = _frameBitmap.Clone(
                new Rectangle(0, 0, _frameBitmap.Width, _frameBitmap.Height),
                PixelFormat.Format32bppRgb);

            _context.Post(state => {
                FrameUpdated.Invoke(frameCopy);
            }, null);
        });
    }

    private uint AlignToBoundary(uint dimension, uint boundary)
    {
        var remainder = dimension % boundary;
        return remainder == 0 ? dimension : dimension + boundary - remainder;
    }

    private void ReleaseResources()
    {
        _memoryAccessor?.Dispose();
        _memoryAccessor = null;
        _memoryFile?.Dispose();
        _memoryFile = null;
        _bitmapContext?.Dispose();
        _bitmapContext = null;
        _frameBitmap?.Dispose();
        _frameBitmap = null;
    }
}

Control Integration

public partial class VideoDisplayControl : VideoPlayerBase
{
    public override Control Viewport => pictureBox;
    
    private readonly VideoFrameProvider _frameProvider;

    public VideoDisplayControl()
    {
        InitializeComponent();
        _frameProvider = new VideoFrameProvider(MediaPlayer, SynchronizationContext.Current);
        _frameProvider.FrameReady += ProcessFrame;
        _frameProvider.FrameUpdated += UpdateFrame;
    }

    protected override void OnLoad(object sender, EventArgs e)
    {
        base.OnLoad(sender, e);
        pictureBox.SizeMode = PictureBoxSizeMode.StretchImage;
        pictureBox.Dock = DockStyle.Fill;
    }

    protected override void OnDestroyed(object sender, EventArgs args)
    {
        base.OnDestroyed(sender, args);
        _frameProvider.ReleaseResources();
    }

    private void ProcessFrame(Graphics canvas)
    {
        canvas.DrawString("Overlay Text", new Font("Arial", 24), Brushes.Red, 100, 100);
    }

    private void UpdateFrame(Bitmap frame)
    {
        var previous = pictureBox.Image;
        pictureBox.Image = frame;
        previous?.Dispose();
    }
}

Performance Optimization

Testing revealed ReadArray operations consumed approximately 100ms for 1080p 30fps video. Replacing this with Marshal.Copy reduced processing time to around 25ms:

private unsafe void RenderFrame(IntPtr userData, IntPtr picture)
{
    if (_frameBitmap == null) return;

    try
    {
        var timer = Stopwatch.StartNew();
        var bitmapData = _frameBitmap.LockBits(
            new Rectangle(0, 0, _videoWidth, _videoHeight),
            ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb);
        
        byte* bufferPtr = (byte*)0;
        _memoryAccessor.SafeMemoryMappedViewHandle.AcquirePointer(ref bufferPtr);
        Marshal.Copy(IntPtr.Add(new IntPtr(bufferPtr), 0), _pixelBuffer, 0, _pixelBuffer.Length);
        Marshal.Copy(_pixelBuffer, 0, bitmapData.Scan0, _pixelBuffer.Length);
        
        _frameBitmap.UnlockBits(bitmapData);
        timer.Stop();
        Debug.WriteLine($"Processing time: {timer.ElapsedMilliseconds}ms");

        FrameReady.Invoke(_bitmapContext);
    }
    catch (Exception ex)
    {
        Console.WriteLine(ex.Message);
    }

    Task.Run(() => {
        var frameCopy = _frameBitmap.Clone(
            new Rectangle(0, 0, _frameBitmap.Width, _frameBitmap.Height),
            PixelFormat.Format32bppRgb);

        _context.Post(state => {
            FrameUpdated.Invoke(frameCopy);
        }, null);
    });
}