Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re-enable and polish IAudioClient3 to achieve lower latencies #740

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+184 −62
Open

Re-enable and polish IAudioClient3 to achieve lower latencies #740

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 184 additions & 62 deletions src/cubeb_wasapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,31 @@
#include "cubeb_tracing.h"
#include "cubeb_utils.h"

// Some people have reported glitches with IAudioClient3 capture streams:
// http://blog.nirbheek.in/2018/03/low-latency-audio-on-windows-with.html
// https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
#define ALLOW_AUDIO_CLIENT_3_FOR_INPUT 0
// IAudioClient3::GetSharedModeEnginePeriod() seem to return min latencies
// bigger than IAudioClient::GetDevicePeriod(), which is confusing (10ms vs
// 3ms), though the default latency is usually the same and we should use the
// IAudioClient3 function anyway, as it's more correct
#define USE_AUDIO_CLIENT_3_MIN_PERIOD 1
// If this is true, we allow IAudioClient3 the creation of sessions with a
// latency above the default one (usually 10ms).
// Whether we should default this to true or false depend on many things:
// -Does creating a shared IAudioClient3 session (not locked to a format)
// actually forces all the IAudioClient(1) sessions to have the same latency?
// I could find no proof of that.
// -Does creating a shared IAudioClient3 session with a latency >= the default
// one actually improve the latency (as in how late the audio is) at all?
// -Maybe we could expose this as cubeb stream pref
// (e.g. take priority over other apps)?
#define ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT 1
// If this is true and the user specified a target latency >= the IAudioClient3
// max one, then we reject it and fall back to IAudioClient(1). There wouldn't
// be much point in having a low latency if that's not what the user wants.
#define REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX 0

// Windows 10 exposes the IAudioClient3 interface to create low-latency streams.
// Copy the interface definition from audioclient.h here to make the code
// simpler and so that we can still access IAudioClient3 via COM if cubeb was
Expand Down Expand Up @@ -1867,6 +1892,44 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
return CUBEB_ERROR;
}

#if USE_AUDIO_CLIENT_3_MIN_PERIOD
// This is unreliable as we can't know the actual mixer format cubeb will
// ask for later on (nor we can branch on ALLOW_AUDIO_CLIENT_3_FOR_INPUT),
// and the min latency can change based on that.
com_ptr<IAudioClient3> client3;
hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
client3.receive_vpp());
if (SUCCEEDED(hr)) {
WAVEFORMATEX * mix_format = nullptr;
hr = client3->GetMixFormat(&mix_format);

if (SUCCEEDED(hr)) {
uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
max_period = 0;
hr = client3->GetSharedModeEnginePeriod(mix_format, &default_period,
&fundamental_period, &min_period,
&max_period);

auto sample_rate = mix_format->nSamplesPerSec;
CoTaskMemFree(mix_format);
if (SUCCEEDED(hr)) {
// Print values in the same format as IAudioDevice::GetDevicePeriod()
REFERENCE_TIME min_period_rt(frames_to_hns(sample_rate, min_period));
REFERENCE_TIME default_period_rt(
frames_to_hns(sample_rate, default_period));
LOG("default device period: %I64d, minimum device period: %I64d",
default_period_rt, min_period_rt);

*latency_frames = min_period;

LOG("Minimum latency in frames: %u", *latency_frames);

return CUBEB_OK;
}
}
}
#endif

com_ptr<IAudioClient> client;
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
client.receive_vpp());
Expand All @@ -1886,18 +1949,8 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
LOG("default device period: %I64d, minimum device period: %I64d",
default_period, minimum_period);

/* If we're on Windows 10, we can use IAudioClient3 to get minimal latency.
Otherwise, according to the docs, the best latency we can achieve is by
synchronizing the stream and the engine.
http://msdn.microsoft.com/en-us/library/windows/desktop/dd370871%28v=vs.85%29.aspx
*/

// #ifdef _WIN32_WINNT_WIN10
#if 0
*latency_frames = hns_to_frames(params.rate, minimum_period);
#else
// The minimum_period is only relevant in exclusive streams.
*latency_frames = hns_to_frames(params.rate, default_period);
#endif

LOG("Minimum latency in frames: %u", *latency_frames);

Expand Down Expand Up @@ -1987,7 +2040,10 @@ handle_channel_layout(cubeb_stream * stm, EDataFlow direction,
if (hr == S_FALSE) {
/* Channel layout not supported, but WASAPI gives us a suggestion. Use it,
and handle the eventual upmix/downmix ourselves. Ignore the subformat of
the suggestion, since it seems to always be IEEE_FLOAT. */
the suggestion, since it seems to always be IEEE_FLOAT.
This fallback doesn't update the bit depth, so if a device
only supported bit depths cubeb doesn't support, so IAudioClient3
streams might fail */
LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
WAVEFORMATEXTENSIBLE * closest_pcm =
Expand Down Expand Up @@ -2031,12 +2087,12 @@ initialize_iaudioclient2(com_ptr<IAudioClient> & audio_client)
return CUBEB_OK;
}

#if 0
bool
initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
cubeb_stream * stm,
const com_heap_ptr<WAVEFORMATEX> & mix_format,
DWORD flags, EDataFlow direction)
DWORD flags, EDataFlow direction,
REFERENCE_TIME latency_hns)
{
com_ptr<IAudioClient3> audio_client3;
audio_client->QueryInterface<IAudioClient3>(audio_client3.receive());
Expand All @@ -2052,24 +2108,22 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
return false;
}

// Some people have reported glitches with capture streams:
// http://blog.nirbheek.in/2018/03/low-latency-audio-on-windows-with.html
if (direction == eCapture) {
LOG("Audio stream is capture, not using IAudioClient3");
return false;
}

// Possibly initialize a shared-mode stream using IAudioClient3. Initializing
// a stream this way lets you request lower latencies, but also locks the
// global WASAPI engine at that latency.
// - If we request a shared-mode stream, streams created with IAudioClient
// will
// have their latency adjusted to match. When the shared-mode stream is
// closed, they'll go back to normal.
// - If there's already a shared-mode stream running, then we cannot request
// the engine change to a different latency - we have to match it.
// - It's antisocial to lock the WASAPI engine at its default latency. If we
// would do this, then stop and use IAudioClient instead.
// might have their latency adjusted to match. When the shared-mode stream
// is closed, they'll go back to normal.
// - If there's already a shared-mode stream running, if it created with the
// AUDCLNT_STREAMOPTIONS_MATCH_FORMAT option, the audio engine would be
// locked to that format, so we have to match it (a custom one would fail).
// - We don't lock the WASAPI engine to a format, as it's antisocial towards
// other apps, especially if we locked to a latency >= than its default.
// - If the user requested latency is >= the default one, we might still
// accept it (without locking the format) depending on
// ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT, as we might want to prioritize
// to lower our latency over other apps
// (there might still be latency advantages compared to IAudioDevice(1)).

HRESULT hr;
uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
Expand All @@ -2081,28 +2135,59 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
LOG("Could not get shared mode engine period: error: %lx", hr);
return false;
}
uint32_t requested_latency = stm->latency;
uint32_t requested_latency =
hns_to_frames(mix_format->nSamplesPerSec, latency_hns);
#if !ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT
if (requested_latency >= default_period) {
LOG("Requested latency %i greater than default latency %i, not using "
"IAudioClient3",
LOG("Requested latency %i equal or greater than default latency %i,"
" not using IAudioClient3",
requested_latency, default_period);
return false;
}
#elif REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX
if (requested_latency > max_period) {
// Fallback to IAudioClient(1) as it's more accepting of large latencies
LOG("Requested latency %i greater than max latency %i,"
" not using IAudioClient3",
requested_latency, max_period);
return false;
}
#endif
LOG("Got shared mode engine period: default=%i fundamental=%i min=%i max=%i",
default_period, fundamental_period, min_period, max_period);
// Snap requested latency to a valid value
uint32_t old_requested_latency = requested_latency;
// The period is required to be a multiple of the fundamental period
// (and >= min and <= max, which should still be true)
requested_latency -= requested_latency % fundamental_period;
if (requested_latency < min_period) {
requested_latency = min_period;
}
requested_latency -= (requested_latency - min_period) % fundamental_period;
// Likely unnecessary, but won't hurt
if (requested_latency > max_period) {
requested_latency = max_period;
}
if (requested_latency != old_requested_latency) {
LOG("Requested latency %i was adjusted to %i", old_requested_latency,
requested_latency);
}

hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
DWORD new_flags = flags;
// Always add these flags to IAudioClient3, they might help
// if the stream doesn't have the same format as the audio engine.
new_flags |= AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM;
new_flags |= AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY;

hr = audio_client3->InitializeSharedAudioStream(new_flags, requested_latency,
mix_format.get(), NULL);
// This error should be returned first even if
// the period was locked (AUDCLNT_E_ENGINE_PERIODICITY_LOCKED)
if (hr == AUDCLNT_E_INVALID_STREAM_FLAG) {
LOG("Got AUDCLNT_E_INVALID_STREAM_FLAG, removing some flags");
hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
mix_format.get(), NULL);
}

if (SUCCEEDED(hr)) {
return true;
} else if (hr == AUDCLNT_E_ENGINE_PERIODICITY_LOCKED) {
Expand All @@ -2114,22 +2199,37 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
}

uint32_t current_period = 0;
WAVEFORMATEX * current_format = nullptr;
WAVEFORMATEX * current_format_ptr = nullptr;
// We have to pass a valid WAVEFORMATEX** and not nullptr, otherwise
// GetCurrentSharedModeEnginePeriod will return E_POINTER
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format,
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format_ptr,
&current_period);
CoTaskMemFree(current_format);
if (FAILED(hr)) {
LOG("Could not get current shared mode engine period: error: %lx", hr);
return false;
}
com_heap_ptr<WAVEFORMATEX> current_format(current_format_ptr);
if (current_format->nSamplesPerSec != mix_format->nSamplesPerSec) {
// Unless some other external app locked the shared mode engine period
// within our audio initialization, this is unlikely to happen, though we
// can't respect the user selected latency, so we fallback on IAudioClient
LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a "
"different mixer format (nSamplesPerSec) from "
"IAudioClient::GetMixFormat(); not using IAudioClient3");
return false;
}

if (current_period >= default_period) {
LOG("Current shared mode engine period %i too high, not using IAudioClient",
current_period);
#if REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX
// Reject IAudioClient3 if we can't respect the user target latency.
// We don't need to check against default_latency anymore,
// as the current_period is already the best one we could get.
if (old_requested_latency > current_period) {
LOG("Requested latency %i greater than currently locked shared mode "
"latency %i, not using IAudioClient3",
old_requested_latency, current_period);
return false;
}
#endif

hr = audio_client3->InitializeSharedAudioStream(flags, current_period,
mix_format.get(), NULL);
Expand All @@ -2142,7 +2242,6 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
LOG("Could not initialize shared stream with IAudioClient3: error: %lx", hr);
return false;
}
#endif

#define DIRECTION_NAME (direction == eCapture ? "capture" : "render")

Expand All @@ -2166,6 +2265,12 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
return CUBEB_ERROR;
}

#if ALLOW_AUDIO_CLIENT_3_FOR_INPUT
constexpr bool allow_audio_client_3 = true;
#else
const bool allow_audio_client_3 = direction == eRender;
#endif

stm->stream_reset_lock.assert_current_thread_owns();
// If user doesn't specify a particular device, we can choose another one when
// the given devid is unavailable.
Expand Down Expand Up @@ -2202,17 +2307,14 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,

/* Get a client. We will get all other interfaces we need from
* this pointer. */
#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
hr = device->Activate(__uuidof(IAudioClient3),
CLSCTX_INPROC_SERVER,
NULL, audio_client.receive_vpp());
if (hr == E_NOINTERFACE) {
#endif
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
audio_client.receive_vpp());
#if 0
if (allow_audio_client_3) {
hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
audio_client.receive_vpp());
}
if (!allow_audio_client_3 || hr == E_NOINTERFACE) {
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
audio_client.receive_vpp());
}
#endif

if (FAILED(hr)) {
LOG("Could not activate the device to get an audio"
Expand Down Expand Up @@ -2341,16 +2443,15 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
}
}

#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
if (initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction)) {
if (allow_audio_client_3 &&
initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction,
latency_hns)) {
LOG("Initialized with IAudioClient3");
} else {
#endif
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0,
mix_format.get(), NULL);
#if 0
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns,
0, mix_format.get(), NULL);
}
#endif

if (FAILED(hr)) {
LOG("Unable to initialize audio client for %s: %lx.", DIRECTION_NAME, hr);
return CUBEB_ERROR;
Expand Down Expand Up @@ -3310,6 +3411,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
CUBEB_DEVICE_FMT_S16NE);
ret.default_format = CUBEB_DEVICE_FMT_F32NE;
prop_variant fmtvar;
WAVEFORMATEX * wfx = NULL;
hr = propstore->GetValue(PKEY_AudioEngine_DeviceFormat, &fmtvar);
if (SUCCEEDED(hr) && fmtvar.vt == VT_BLOB) {
if (fmtvar.blob.cbSize == sizeof(PCMWAVEFORMAT)) {
Expand All @@ -3319,8 +3421,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
ret.max_rate = ret.min_rate = ret.default_rate = pcm->wf.nSamplesPerSec;
ret.max_channels = pcm->wf.nChannels;
} else if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX)) {
WAVEFORMATEX * wfx =
reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
wfx = reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);

if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX) + wfx->cbSize ||
wfx->wFormatTag == WAVE_FORMAT_PCM) {
Expand All @@ -3330,9 +3431,30 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
}
}

if (SUCCEEDED(dev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER,
NULL, client.receive_vpp())) &&
SUCCEEDED(client->GetDevicePeriod(&def_period, &min_period))) {
#if USE_AUDIO_CLIENT_3_MIN_PERIOD
// Here we assume an IAudioClient3 stream will successfully
// be initialized later (it might fail)
#if ALLOW_AUDIO_CLIENT_3_FOR_INPUT
constexpr bool allow_audio_client_3 = true;
#else
const bool allow_audio_client_3 = flow == eRender;
#endif
com_ptr<IAudioClient3> client3;
uint32_t def, fun, min, max;
if (allow_audio_client_3 && wfx &&
SUCCEEDED(dev->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER,
NULL, client3.receive_vpp())) &&
SUCCEEDED(
client3->GetSharedModeEnginePeriod(wfx, &def, &fun, &min, &max))) {
ret.latency_lo = min;
// This latency might actually be used as "default" and not "max" later on,
// so we return the default (we never really want to use the max anyway)
ret.latency_hi = def;
} else
#endif
if (SUCCEEDED(dev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER,
NULL, client.receive_vpp())) &&
SUCCEEDED(client->GetDevicePeriod(&def_period, &min_period))) {
ret.latency_lo = hns_to_frames(ret.default_rate, min_period);
ret.latency_hi = hns_to_frames(ret.default_rate, def_period);
} else {
Expand Down