I try to implement processing frames from webcam to the WPF application using UWP API. There is article how to work with MediaCapture & MediaFrameReader: https://docs.microsoft.com/en-us/windows/uwp/audio-video-camera/process-media-frames-with-mediaframereader#handle-the-frame-arrived-event
If I set up MemoryPreference to cpu, SoftwareBitmaps are initialized to the null in the event. When I place Auto, I can see IDirect3DSurface objects are in the event, but in conversion to the SoftwareBitmap the exception "Specified cast is not valid." is raised.
How to convert IDirect3DSurface to SoftwareBitmap?
private async void MediaCaptureExample()
{
var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync();
MediaFrameSourceGroup selectedGroup = null;
MediaFrameSourceInfo colorSourceInfo = null;
foreach (var sourceGroup in frameSourceGroups)
{
foreach (var sourceInfo in sourceGroup.SourceInfos)
{
if (sourceInfo.MediaStreamType == MediaStreamType.VideoRecord && sourceInfo.SourceKind == MediaFrameSourceKind.Color)
{
colorSourceInfo = sourceInfo;
break;
}
}
if (colorSourceInfo != null)
{
selectedGroup = sourceGroup;
break;
}
}
capture = new MediaCapture();
var settings = new MediaCaptureInitializationSettings()
{
SourceGroup = selectedGroup,
SharingMode = MediaCaptureSharingMode.ExclusiveControl,
MemoryPreference = MediaCaptureMemoryPreference.Auto,
StreamingCaptureMode = StreamingCaptureMode.Video
};
await capture.InitializeAsync(settings);
var colorFrameSource = capture.FrameSources[colorSourceInfo.Id];
var preferredFormat = colorFrameSource.SupportedFormats.Where(format =
{
return format.VideoFormat.Width = 1080
&& String.Compare(format.Subtype, MediaEncodingSubtypes.Mjpg, true) == 0;
}).FirstOrDefault();
if (preferredFormat == null)
{
// Our desired format is not supported
return;
}
await colorFrameSource.SetFormatAsync(preferredFormat);
mediaFrameReader = await capture.CreateFrameReaderAsync(colorFrameSource);
mediaFrameReader.FrameArrived += MediaFrameReader_FrameArrived;
var result = await mediaFrameReader.StartAsync();
Console.WriteLine("Result = " + result.ToString());
}
private void MediaFrameReader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{
try
{
var mediaFrameReference = sender.TryAcquireLatestFrame();
var videoMediaFrame = mediaFrameReference?.VideoMediaFrame;
var softwareBitmap = videoMediaFrame?.SoftwareBitmap;
var direct3DSurface = videoMediaFrame?.Direct3DSurface;
if (direct3DSurface != null)
{
var softwareBitmapTask = SoftwareBitmap.CreateCopyFromSurfaceAsync(mediaFrameReference.VideoMediaFrame.Direct3DSurface).AsTask();
softwareBitmap = softwareBitmapTask.Result;
}
if (softwareBitmap != null)
{
using (var stream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
var encoderTask = BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream).AsTask();
encoderTask.Wait();
var encoder = encoderTask.Result;
encoder.SetSoftwareBitmap(softwareBitmap);
Task t = encoder.FlushAsync().AsTask();
t.Wait();
var image = new System.Windows.Media.Imaging.BitmapImage();
image.BeginInit();
image.StreamSource = stream.AsStream();
image.CacheOption = System.Windows.Media.Imaging.BitmapCacheOption.OnLoad;
image.EndInit();
imageElement.Source = image;
}
}
}
catch(Exception e)
{
Console.WriteLine(e.Message);
}
}