Here's my code snippet, on Windows 10. If the <width, height> is set to other resolution such as 1280x760, 640x480, the pDecoderTransform_->SetOutputType() would fail with such error.
E [....] Failed to set output media type on H.264 decoder MFT.
Reason: The data specified for the media type is invalid, inconsistent, or not supported by this object. (Code: C00D36B4)
Please help.
hr = CoCreateInstance(CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IUnknown, (void**) &pTransformUnk);
CHECK_HR_RET_BOOL(hr, "Failed to create H264 decoder MFT.");
hr = pTransformUnk->QueryInterface(IID_PPV_ARGS(&pDecoderTransform_));
CHECK_HR_RET_BOOL(hr, "Failed to get IMFTransform interface from H264 decoder MFT object.");
IMFAttributes* decoderAttributes;
hr = pDecoderTransform_->GetAttributes(&decoderAttributes);
CHECK_HR_RET_BOOL(hr, "Failed to get attributes.");
hr = decoderAttributes->SetUINT32(CODECAPI_AVDecVideoAcceleration_H264, TRUE);
CHECK_HR_RET_BOOL(hr, "Failed to enable CODECAPI_AVDecVideoAcceleration_H264");
MFCreateMediaType (&pInputMediaType);
pInputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
pInputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
hr = pDecoderTransform_->SetInputType(0, pInputMediaType, 0);
CHECK_HR_RET_BOOL(hr, "Failed to set input media type on decoder MFT.");
MFCreateMediaType (&pOutputMediaType);
pOutputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
pOutputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_IYUV);
// In the following lines, if other resolution is specified, e.g. 1280x760, 640x480,
// "pDecoderTransform_->SetOutputType()" would fail.
width = 1920;
height = 1080;
hr = MFSetAttributeSize(pOutputMediaType, MF_MT_FRAME_SIZE, width, height);
CHECK_HR_RET_BOOL(hr, "Failed to set frame size on H264 MFT out type.");
hr = pDecoderTransform_->SetOutputType(0, pOutputMediaType, 0);
CHECK_HR_RET_BOOL(hr, "Failed to set output media type on H.264 decoder MFT.");