Skip to content

Commit

Permalink
cubeb - re-enable and polish IAudioClient3 to achieve lower latencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@Filoppi
Filoppi committed Jan 20, 2023
1 parent 27d2a10 commit 6c79b87
Showing 1 changed file with 191 additions and 39 deletions.
230 changes: 191 additions & 39 deletions src/cubeb_wasapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@
#define _WIN32_WINNT 0x0603
#define NOMINMAX

// Makes sure cubeb doesn't try to initialize an IAudioClient3 with unsupported formats, as it forces its own (e.g. unsupported bit depths or sample rates).
// I'm not 100% sure this is right as IAudioClient::IsFormatSupported() might not be relevant with IAudioClient3, but it probably is.
#define CHECK_MIXER_FORMAT_SUPPORT 1
// Force raw output (skips some processing) with IAudioClient3
#define USE_RAW_OUTPUT 0
// Forces the output format to match our mixer one
#define FORCE_MATCHING_OUTPUT_FORMAT 0
// "Forces" (?) a IAudioClient3 to resample (in high quality) our stream to the device output sample rate
#define CONVERT_STREAM 0
// If we are targeting IAudioClient3, we don't want to limit the latency to the default latency (likely 10ms), but to the minimum one.
// Though it seems that IAudioClient::GetDevicePeriod() returns lower latencies than GetSharedModeEnginePeriod() allows, so that's weird.
#define ALLOW_MIN_LATENCY 0
// Can we go lower than the user/client requested latency? This isn't particularly necessary as dolphin should produce enough audio samples to go with latencies < 10ms (which is what it asks for),
// and the user requested latency is just a suggestion.
#define FORCE_RESPECT_USER_LATENCY 0
// IAudioClient3::GetSharedModeEnginePeriod() doesn't seem to work as well as IAudioClient::GetDevicePeriod() and often return a min and max latency of 10ms, even if lower ones would be supported
#define USE_AUDIO_CLIENT_3_LATENCY 0

#include <algorithm>
#include <atomic>
#include <audioclient.h>
Expand Down Expand Up @@ -1867,6 +1885,45 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
return CUBEB_ERROR;
}

#if USE_AUDIO_CLIENT_3_LATENCY
//TODO: this is likely unreliable as we can't know the actual mixer format cubeb will ask for later on (we'd need to calculate it) (and the min latency could change based on that)
com_ptr<IAudioClient3> client3;
hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
client3.receive_vpp());
if (SUCCEEDED(hr)) {
WAVEFORMATEX * mix_format = nullptr;
hr = client3->GetMixFormat(&mix_format);

if (SUCCEEDED(hr)) {
uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
max_period = 0;
hr = client3->GetSharedModeEnginePeriod(
mix_format, &default_period, &fundamental_period, &min_period,
&max_period);

auto sample_rate = mix_format->nSamplesPerSec;
CoTaskMemFree(mix_format);
if (SUCCEEDED(hr)) {
// Keep values in the same format as IAudioDevice::GetDevicePeriod()
REFERENCE_TIME min_period_rt(frames_to_hns(sample_rate, min_period));
REFERENCE_TIME default_period_rt(frames_to_hns(sample_rate, default_period));
LOG("default device period: %I64d, minimum device period: %I64d",
default_period_rt, min_period_rt);

#if ALLOW_MIN_LATENCY
*latency_frames = min_period;
#else
*latency_frames = default_period; // This can be 0
#endif

LOG("Minimum latency in frames: %u", *latency_frames);

return CUBEB_OK;
}
}
}
#endif

com_ptr<IAudioClient> client;
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
client.receive_vpp());
Expand All @@ -1891,10 +1948,8 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
synchronizing the stream and the engine.
http://msdn.microsoft.com/en-us/library/windows/desktop/dd370871%28v=vs.85%29.aspx
*/

// #ifdef _WIN32_WINNT_WIN10
#if 0
*latency_frames = hns_to_frames(params.rate, minimum_period);
#if defined(_WIN32_WINNT_WIN10) && ALLOW_MIN_LATENCY
*latency_frames = hns_to_frames(params.rate, minimum_period);
#else
*latency_frames = hns_to_frames(params.rate, default_period);
#endif
Expand Down Expand Up @@ -1987,7 +2042,10 @@ handle_channel_layout(cubeb_stream * stm, EDataFlow direction,
if (hr == S_FALSE) {
/* Channel layout not supported, but WASAPI gives us a suggestion. Use it,
and handle the eventual upmix/downmix ourselves. Ignore the subformat of
the suggestion, since it seems to always be IEEE_FLOAT. */
the suggestion, since it seems to always be IEEE_FLOAT.
This fallback doesn't update the bit depth, so if a device
only supported bit depths cubeb doesn't support, we will need to
make sure the stream is compatible with it (e.g. IAudioClient3 is not) */
LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
WAVEFORMATEXTENSIBLE * closest_pcm =
Expand Down Expand Up @@ -2031,12 +2089,12 @@ initialize_iaudioclient2(com_ptr<IAudioClient> & audio_client)
return CUBEB_OK;
}

#if 0
bool
initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
cubeb_stream * stm,
const com_heap_ptr<WAVEFORMATEX> & mix_format,
DWORD flags, EDataFlow direction)
DWORD flags, EDataFlow direction,
REFERENCE_TIME latency_hns)
{
com_ptr<IAudioClient3> audio_client3;
audio_client->QueryInterface<IAudioClient3>(audio_client3.receive());
Expand Down Expand Up @@ -2072,37 +2130,98 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
// would do this, then stop and use IAudioClient instead.

HRESULT hr;

#if CHECK_MIXER_FORMAT_SUPPORT
WAVEFORMATEX * tmp = nullptr;
// The mixer format here might have bit depths "forced" by cubeb even if the output device doesn't directly support it.
// This doesn't work with IAudioClient3 as there it applies no resampling.
hr = audio_client3->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED, mix_format.get(), &tmp);
if (FAILED(hr)) {
LOG("IAudioClient3 attempted format is not supported: error: %lx", hr);
return false;
}
CoTaskMemFree(tmp);
#endif

uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
max_period = 0;
//TODO: review... this returns the same default/min/max latency every time on my PC (10ms, which is high),
//independently of the device and its settings (with no other apps running). Not sure if it means that lag would be bigger than with IAudioDevice(1).
hr = audio_client3->GetSharedModeEnginePeriod(
mix_format.get(), &default_period, &fundamental_period, &min_period,
&max_period);
if (FAILED(hr)) {
LOG("Could not get shared mode engine period: error: %lx", hr);
return false;
}
uint32_t requested_latency = stm->latency;
if (requested_latency >= default_period) {
LOG("Requested latency %i greater than default latency %i, not using "
uint32_t requested_latency = hns_to_frames(mix_format->nSamplesPerSec, latency_hns);
#if FORCE_RESPECT_USER_LATENCY
if (requested_latency > max_period) {
// Fallback to IAudioClient(1) as it's less restrictive towards bigger latencies
LOG("Requested latency %i greater than max latency %i, not using "
"IAudioClient3",
requested_latency, default_period);
requested_latency, max_period);
return false;
}
#endif
LOG("Got shared mode engine period: default=%i fundamental=%i min=%i max=%i",
default_period, fundamental_period, min_period, max_period);
// Snap requested latency to a valid value
uint32_t old_requested_latency = requested_latency;
// The period is required to be a multiple of the fundamental period (and >= min and <= max, which should still be true)
requested_latency -= requested_latency % fundamental_period;
if (requested_latency < min_period) {
requested_latency = min_period;
}
requested_latency -= (requested_latency - min_period) % fundamental_period;
// This is likely unnecessary, but it won't hurt
if (requested_latency > max_period) {
requested_latency = max_period;
}
if (requested_latency != old_requested_latency) {
LOG("Requested latency %i was adjusted to %i", old_requested_latency,
requested_latency);
}

hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
#if USE_RAW_OUTPUT || FORCE_MATCHING_OUTPUT_FORMAT
AudioClientProperties properties = {0};
properties.cbSize = sizeof(AudioClientProperties);
properties.bIsOffload = false; //TODO: review this (take from chromium)
#ifndef __MINGW32__
#if USE_RAW_OUTPUT
// Raw audio streams skip some kinds of processing, like AEC and AGC.
// Do this independent from CUBEB_STREAM_PREF_RAW, we force this with IAudioClient3, to hopefully get the lowest latency/quality and support.
properties.Options |= AUDCLNT_STREAMOPTIONS_RAW;
#endif
#if FORCE_MATCHING_OUTPUT_FORMAT
properties.Options |= AUDCLNT_STREAMOPTIONS_MATCH_FORMAT;
#endif
#endif
com_ptr<IAudioClient2> audio_client2;
hr = audio_client->QueryInterface<IAudioClient2>(audio_client2.receive());
if (audio_client2) {
hr = audio_client2->SetClientProperties(&properties);
}
if (FAILED(hr)) {
LOG("Could not set IAudioClient2 properties: error: %lx", hr);
// This is not fatal, it should work anyway (AUDCLNT_STREAMOPTIONS_RAW might not be supported)
}
#endif

DWORD new_flags = flags;
#if CONVERT_STREAM
// Always add these flags to IAudioClient3 (they can help if the stream doesn't have the same format as the output)
new_flags |= AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM;
new_flags |= AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY;
#endif

hr = audio_client3->InitializeSharedAudioStream(new_flags, requested_latency,
mix_format.get(), NULL);
if (hr == AUDCLNT_E_INVALID_STREAM_FLAG && CONVERT_STREAM) {
LOG("Got AUDCLNT_E_INVALID_STREAM_FLAG, removing some flag");
hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
mix_format.get(), NULL);
}

if (SUCCEEDED(hr)) {
return true;
} else if (hr == AUDCLNT_E_ENGINE_PERIODICITY_LOCKED) {
Expand All @@ -2114,22 +2233,48 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
}

uint32_t current_period = 0;
WAVEFORMATEX * current_format = nullptr;
WAVEFORMATEX * current_format_ptr = nullptr;
// We have to pass a valid WAVEFORMATEX** and not nullptr, otherwise
// GetCurrentSharedModeEnginePeriod will return E_POINTER
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format,
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format_ptr,
&current_period);
CoTaskMemFree(current_format);
if (FAILED(hr)) {
LOG("Could not get current shared mode engine period: error: %lx", hr);
return false;
}
com_heap_ptr<WAVEFORMATEX> current_format(current_format_ptr);
#if 1
// Unless some other external app locked the shared mode engine period within our audio initialization,
// this likely shouldn't happen
if (current_format->nSamplesPerSec != mix_format->nSamplesPerSec)
{
hr = audio_client3->GetSharedModeEnginePeriod(
current_format.get(), &default_period, &fundamental_period, &min_period,
&max_period);
if (FAILED(hr)) {
LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a different mixer format (nSamplesPerSec) from IAudioClient::GetMixFormat(); not using IAudioClient3");
return false;
}
LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a different mixer format (nSamplesPerSec) from IAudioClient::GetMixFormat(); attempting a matching latency");
REFERENCE_TIME current_period_hns = frames_to_hns(current_format->nSamplesPerSec, current_period);
current_period = hns_to_frames(mix_format->nSamplesPerSec, current_period_hns);
current_period -= current_period % fundamental_period;
// Note: the follow up IAudioClient3::InitializeSharedAudioStream() is likely to fail anyway,
// given that the period was already locked and we changed it.
// The only right way to proceed would be to change the "mix_format->nSamplesPerSec" to "current_format->nSamplesPerSec",
// but we can't do that for now.
}
#endif

if (current_period >= default_period) {
LOG("Current shared mode engine period %i too high, not using IAudioClient",
current_period);
#if FORCE_RESPECT_USER_LATENCY
// This used to return false if current_period was >= default_latency, but that didn't seem to make sense
if (old_requested_latency > current_period) {
LOG("Requested latency %i greater than currently locked shared mode latency %i, not using "
"IAudioClient3",
old_requested_latency, current_period);
return false;
}
#endif

hr = audio_client3->InitializeSharedAudioStream(flags, current_period,
mix_format.get(), NULL);
Expand All @@ -2142,7 +2287,6 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
LOG("Could not initialize shared stream with IAudioClient3: error: %lx", hr);
return false;
}
#endif

#define DIRECTION_NAME (direction == eCapture ? "capture" : "render")

Expand All @@ -2166,6 +2310,8 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
return CUBEB_ERROR;
}

const bool has_capture = direction == eCapture || direction == eAll;

stm->stream_reset_lock.assert_current_thread_owns();
// If user doesn't specify a particular device, we can choose another one when
// the given devid is unavailable.
Expand Down Expand Up @@ -2202,17 +2348,17 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,

/* Get a client. We will get all other interfaces we need from
* this pointer. */
#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
hr = device->Activate(__uuidof(IAudioClient3),
CLSCTX_INPROC_SERVER,
NULL, audio_client.receive_vpp());
if (hr == E_NOINTERFACE) {
#endif
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
audio_client.receive_vpp());
#if 0
if (!has_capture) {
hr = device->Activate(__uuidof(IAudioClient3),
CLSCTX_INPROC_SERVER,
NULL, audio_client.receive_vpp());
}
// IAudioClient3 has problems with capture sessions:
// https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
if (has_capture || hr == E_NOINTERFACE) {
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
audio_client.receive_vpp());
}
#endif

if (FAILED(hr)) {
LOG("Could not activate the device to get an audio"
Expand Down Expand Up @@ -2341,16 +2487,12 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
}
}

#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
if (initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction)) {
if (!has_capture && initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction, latency_hns)) {
LOG("Initialized with IAudioClient3");
} else {
#endif
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0,
mix_format.get(), NULL);
#if 0
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0, mix_format.get(), NULL);
}
#endif

if (FAILED(hr)) {
LOG("Unable to initialize audio client for %s: %lx.", DIRECTION_NAME, hr);
return CUBEB_ERROR;
Expand Down Expand Up @@ -3310,6 +3452,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
CUBEB_DEVICE_FMT_S16NE);
ret.default_format = CUBEB_DEVICE_FMT_F32NE;
prop_variant fmtvar;
WAVEFORMATEX* wfx = NULL;
hr = propstore->GetValue(PKEY_AudioEngine_DeviceFormat, &fmtvar);
if (SUCCEEDED(hr) && fmtvar.vt == VT_BLOB) {
if (fmtvar.blob.cbSize == sizeof(PCMWAVEFORMAT)) {
Expand All @@ -3319,8 +3462,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
ret.max_rate = ret.min_rate = ret.default_rate = pcm->wf.nSamplesPerSec;
ret.max_channels = pcm->wf.nChannels;
} else if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX)) {
WAVEFORMATEX * wfx =
reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
wfx = reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);

if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX) + wfx->cbSize ||
wfx->wFormatTag == WAVE_FORMAT_PCM) {
Expand All @@ -3330,6 +3472,16 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
}
}

#if USE_AUDIO_CLIENT_3_LATENCY
// Here we guess that an IAudioClient3 stream will successfully be initialized later (it might fail).
com_ptr<IAudioClient3> client3;
uint32_t def, fun, min, max;
if (wfx && SUCCEEDED(dev->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL, client3.receive_vpp()))
&& SUCCEEDED(client3->GetSharedModeEnginePeriod(wfx, &def, &fun, &min, &max))) {
ret.latency_lo = min;
ret.latency_hi = def;
} else
#endif
if (SUCCEEDED(dev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER,
NULL, client.receive_vpp())) &&
SUCCEEDED(client->GetDevicePeriod(&def_period, &min_period))) {
Expand Down

0 comments on commit 6c79b87

Please sign in to comment.