diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp --- a/dom/media/GraphDriver.cpp +++ b/dom/media/GraphDriver.cpp @@ -1,1220 +1,1204 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include #include "mozilla/dom/AudioContext.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/Unused.h" #include "CubebUtils.h" #ifdef MOZ_WEBRTC #include "webrtc/MediaEngineWebRTC.h" #endif #ifdef XP_MACOSX #include #endif extern mozilla::LazyLogModule gMediaStreamGraphLog; #ifdef LOG #undef LOG #endif // LOG #define LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg) namespace mozilla { GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl) : mIterationStart(0), mIterationEnd(0), mGraphImpl(aGraphImpl), mCurrentTimeStamp(TimeStamp::Now()), mPreviousDriver(nullptr), - mNextDriver(nullptr), - mScheduled(false) + mNextDriver(nullptr) { } void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver, GraphTime aLastSwitchNextIterationStart, GraphTime aLastSwitchNextIterationEnd) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); // We set mIterationEnd here, because the first thing a driver do when it // does an iteration is to update graph times, so we are in fact setting // mIterationStart of the next iteration by setting the end of the previous // iteration. mIterationStart = aLastSwitchNextIterationStart; mIterationEnd = aLastSwitchNextIterationEnd; MOZ_ASSERT(!PreviousDriver()); MOZ_ASSERT(aPreviousDriver); LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", aPreviousDriver, aPreviousDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver")); SetPreviousDriver(aPreviousDriver); } void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); LOG(LogLevel::Debug, ("Switching to new driver: %p (%s)", aNextDriver, aNextDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver")); if (mNextDriver && mNextDriver != GraphImpl()->CurrentDriver()) { LOG(LogLevel::Debug, ("Discarding previous next driver: %p (%s)", mNextDriver.get(), mNextDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver")); } SetNextDriver(aNextDriver); } GraphTime GraphDriver::StateComputedTime() const { return mGraphImpl->mStateComputedTime; } void GraphDriver::EnsureNextIteration() { mGraphImpl->EnsureNextIteration(); } bool GraphDriver::Switching() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mNextDriver || mPreviousDriver; } GraphDriver* GraphDriver::NextDriver() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mNextDriver; } GraphDriver* GraphDriver::PreviousDriver() { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); return mPreviousDriver; } void GraphDriver::SetNextDriver(GraphDriver* aNextDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); mNextDriver = aNextDriver; } void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) { GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); mPreviousDriver = aPreviousDriver; } -bool GraphDriver::Scheduled() -{ - GraphImpl()->GetMonitor().AssertCurrentThreadOwns(); - return mScheduled; -} - ThreadedDriver::ThreadedDriver(MediaStreamGraphImpl* aGraphImpl) : GraphDriver(aGraphImpl) { } class MediaStreamGraphShutdownThreadRunnable : public Runnable { public: explicit MediaStreamGraphShutdownThreadRunnable( already_AddRefed aThread) : Runnable("MediaStreamGraphShutdownThreadRunnable") , mThread(aThread) { } NS_IMETHOD Run() override { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mThread); mThread->Shutdown(); mThread = nullptr; return NS_OK; } private: nsCOMPtr mThread; }; ThreadedDriver::~ThreadedDriver() { if (mThread) { nsCOMPtr event = new MediaStreamGraphShutdownThreadRunnable(mThread.forget()); GraphImpl()->Dispatch(event.forget()); } } class MediaStreamGraphInitThreadRunnable : public Runnable { public: explicit MediaStreamGraphInitThreadRunnable(ThreadedDriver* aDriver) : Runnable("MediaStreamGraphInitThreadRunnable") , mDriver(aDriver) { } NS_IMETHOD Run() override { LOG(LogLevel::Debug, ("Starting a new system driver for graph %p", mDriver->mGraphImpl)); RefPtr previousDriver; { MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); previousDriver = mDriver->PreviousDriver(); } if (previousDriver) { LOG(LogLevel::Debug, ("%p releasing an AudioCallbackDriver(%p), for graph %p", mDriver.get(), previousDriver.get(), mDriver->GraphImpl())); MOZ_ASSERT(!mDriver->AsAudioCallbackDriver()); RefPtr releaseEvent = new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(); MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); mDriver->SetPreviousDriver(nullptr); } else { MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor()); MOZ_ASSERT(mDriver->mGraphImpl->MessagesQueued() || mDriver->mGraphImpl->mForceShutDown, "Don't start a graph without messages queued."); mDriver->mGraphImpl->SwapMessageQueues(); } mDriver->RunThread(); return NS_OK; } private: RefPtr mDriver; }; void ThreadedDriver::Start() { LOG(LogLevel::Debug, ("Starting thread for a SystemClockDriver %p", mGraphImpl)); Unused << NS_WARN_IF(mThread); if (!mThread) { // Ensure we haven't already started it nsCOMPtr event = new MediaStreamGraphInitThreadRunnable(this); // Note: mThread may be null during event->Run() if we pass to NewNamedThread! See AudioInitTask nsresult rv = NS_NewNamedThread("MediaStreamGrph", getter_AddRefs(mThread)); if (NS_SUCCEEDED(rv)) { - rv = mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL); - mScheduled = NS_SUCCEEDED(rv); + mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL); } } } void ThreadedDriver::Revive() { // Note: only called on MainThread, without monitor // We know were weren't in a running state LOG(LogLevel::Debug, ("AudioCallbackDriver reviving.")); // If we were switching, switch now. Otherwise, tell thread to run the main // loop again. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (NextDriver()) { NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); } else { nsCOMPtr event = new MediaStreamGraphInitThreadRunnable(this); mThread->EventTarget()->Dispatch(event.forget(), NS_DISPATCH_NORMAL); } } void ThreadedDriver::RemoveCallback() { } void ThreadedDriver::Shutdown() { NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread"); // mGraph's thread is not running so it's OK to do whatever here LOG(LogLevel::Debug, ("Stopping threads for MediaStreamGraph %p", this)); if (mThread) { mThread->Shutdown(); mThread = nullptr; } } SystemClockDriver::SystemClockDriver(MediaStreamGraphImpl* aGraphImpl) : ThreadedDriver(aGraphImpl), mInitialTimeStamp(TimeStamp::Now()), mLastTimeStamp(TimeStamp::Now()), mWaitState(WAITSTATE_RUNNING), mIsFallback(false) {} SystemClockDriver::~SystemClockDriver() { } void SystemClockDriver::MarkAsFallback() { mIsFallback = true; } bool SystemClockDriver::IsFallback() { return mIsFallback; } void ThreadedDriver::RunThread() { while (true) { mIterationStart = IterationEnd(); mIterationEnd += GetIntervalForIteration(); GraphTime stateComputedTime = StateComputedTime(); if (stateComputedTime < mIterationEnd) { LOG(LogLevel::Warning, ("Media graph global underrun detected")); mIterationEnd = stateComputedTime; } if (mIterationStart >= mIterationEnd) { NS_ASSERTION(mIterationStart == mIterationEnd , "Time can't go backwards!"); // This could happen due to low clock resolution, maybe? LOG(LogLevel::Debug, ("Time did not advance")); } GraphTime nextStateComputedTime = mGraphImpl->RoundUpToNextAudioBlock( mIterationEnd + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS)); if (nextStateComputedTime < stateComputedTime) { // A previous driver may have been processing further ahead of // iterationEnd. LOG(LogLevel::Warning, ("Prevent state from going backwards. interval[%ld; %ld] state[%ld; " "%ld]", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime)); nextStateComputedTime = stateComputedTime; } LOG(LogLevel::Verbose, ("interval[%ld; %ld] state[%ld; %ld]", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime)); bool stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); if (!stillProcessing) { // Enter shutdown mode. The stable-state handler will detect this // and complete shutdown if the graph does not get restarted. mGraphImpl->SignalMainThreadCleanup(); return; } MonitorAutoLock lock(GraphImpl()->GetMonitor()); if (NextDriver()) { LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver")); RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); return; } } } MediaTime SystemClockDriver::GetIntervalForIteration() { TimeStamp now = TimeStamp::Now(); MediaTime interval = mGraphImpl->SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds()); mCurrentTimeStamp = now; MOZ_LOG(gMediaStreamGraphLog, LogLevel::Verbose, ("Updating current time to %f (real %f, StateComputedTime() %f)", mGraphImpl->MediaTimeToSeconds(IterationEnd() + interval), (now - mInitialTimeStamp).ToSeconds(), mGraphImpl->MediaTimeToSeconds(StateComputedTime()))); return interval; } TimeStamp OfflineClockDriver::GetCurrentTimeStamp() { MOZ_CRASH("This driver does not support getting the current timestamp."); return TimeStamp(); } void SystemClockDriver::WaitForNextIteration() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); PRIntervalTime timeout = PR_INTERVAL_NO_TIMEOUT; TimeStamp now = TimeStamp::Now(); // This lets us avoid hitting the Atomic twice when we know we won't sleep bool another = mGraphImpl->mNeedAnotherIteration; // atomic if (!another) { mGraphImpl->mGraphDriverAsleep = true; // atomic mWaitState = WAITSTATE_WAITING_INDEFINITELY; } // NOTE: mNeedAnotherIteration while also atomic may have changed before // we could set mGraphDriverAsleep, so we must re-test it. // (EnsureNextIteration sets mNeedAnotherIteration, then tests // mGraphDriverAsleep if (another || mGraphImpl->mNeedAnotherIteration) { // atomic int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS - int64_t((now - mCurrentTimeStamp).ToMilliseconds()); // Make sure timeoutMS doesn't overflow 32 bits by waking up at // least once a minute, if we need to wake up at all timeoutMS = std::max(0, std::min(timeoutMS, 60*1000)); timeout = PR_MillisecondsToInterval(uint32_t(timeoutMS)); LOG(LogLevel::Verbose, ("Waiting for next iteration; at %f, timeout=%f", (now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0)); if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) { mGraphImpl->mGraphDriverAsleep = false; // atomic } mWaitState = WAITSTATE_WAITING_FOR_NEXT_ITERATION; } if (timeout > 0) { mGraphImpl->GetMonitor().Wait(timeout); LOG(LogLevel::Verbose, ("Resuming after timeout; at %f, elapsed=%f", (TimeStamp::Now() - mInitialTimeStamp).ToSeconds(), (TimeStamp::Now() - now).ToSeconds())); } if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) { mGraphImpl->mGraphDriverAsleep = false; // atomic } // Note: this can race against the EnsureNextIteration setting // WAITSTATE_RUNNING and setting mGraphDriverAsleep to false, so you can // have an iteration with WAITSTATE_WAKING_UP instead of RUNNING. mWaitState = WAITSTATE_RUNNING; mGraphImpl->mNeedAnotherIteration = false; // atomic } void SystemClockDriver::WakeUp() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); // Note: this can race against the thread setting WAITSTATE_RUNNING and // setting mGraphDriverAsleep to false, so you can have an iteration // with WAITSTATE_WAKING_UP instead of RUNNING. mWaitState = WAITSTATE_WAKING_UP; mGraphImpl->mGraphDriverAsleep = false; // atomic mGraphImpl->GetMonitor().Notify(); } OfflineClockDriver::OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice) : ThreadedDriver(aGraphImpl), mSlice(aSlice) { } OfflineClockDriver::~OfflineClockDriver() { } MediaTime OfflineClockDriver::GetIntervalForIteration() { return mGraphImpl->MillisecondsToMediaTime(mSlice); } void OfflineClockDriver::WaitForNextIteration() { // No op: we want to go as fast as possible when we are offline } void OfflineClockDriver::WakeUp() { MOZ_ASSERT(false, "An offline graph should not have to wake up."); } AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation) : Runnable("AsyncCubebTask") , mDriver(aDriver) , mOperation(aOperation) , mShutdownGrip(aDriver->GraphImpl()) { NS_WARNING_ASSERTION(mDriver->mAudioStream || aOperation == INIT, "No audio stream!"); } AsyncCubebTask::~AsyncCubebTask() { } SharedThreadPool* AudioCallbackDriver::GetInitShutdownThread() { if (!mInitShutdownThread) { mInitShutdownThread = SharedThreadPool::Get(NS_LITERAL_CSTRING("CubebOperation"), 1); const uint32_t kIdleThreadTimeoutMs = 2000; mInitShutdownThread-> SetIdleThreadTimeout(PR_MillisecondsToInterval(kIdleThreadTimeoutMs)); } return mInitShutdownThread; } NS_IMETHODIMP AsyncCubebTask::Run() { MOZ_ASSERT(mDriver); switch(mOperation) { case AsyncCubebOperation::INIT: { LOG(LogLevel::Debug, ("AsyncCubebOperation::INIT driver=%p", mDriver.get())); if (!mDriver->Init()) { return NS_ERROR_FAILURE; } mDriver->CompleteAudioContextOperations(mOperation); break; } case AsyncCubebOperation::SHUTDOWN: { LOG(LogLevel::Debug, ("AsyncCubebOperation::SHUTDOWN driver=%p", mDriver.get())); mDriver->Stop(); mDriver->CompleteAudioContextOperations(mOperation); mDriver = nullptr; mShutdownGrip = nullptr; break; } default: MOZ_CRASH("Operation not implemented."); } // The thread will kill itself after a bit return NS_OK; } StreamAndPromiseForOperation::StreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation) : mStream(aStream) , mPromise(aPromise) , mOperation(aOperation) { // MOZ_ASSERT(aPromise); } AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl) : GraphDriver(aGraphImpl) , mOutputChannels(0) , mSampleRate(0) , mInputChannels(1) , mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS) , mStarted(false) , mAudioInput(nullptr) , mAddedMixer(false) , mInCallback(false) , mMicrophoneActive(false) + , mShouldFallbackIfError(false) , mFromFallback(false) { LOG(LogLevel::Debug, ("AudioCallbackDriver ctor for graph %p", aGraphImpl)); #if defined(XP_WIN) if (XRE_IsContentProcess()) { audio::AudioNotificationReceiver::Register(this); } #endif } AudioCallbackDriver::~AudioCallbackDriver() { MOZ_ASSERT(mPromisesForOperation.IsEmpty()); #if defined(XP_WIN) if (XRE_IsContentProcess()) { audio::AudioNotificationReceiver::Unregister(this); } #endif } bool IsMacbookOrMacbookAir() { #ifdef XP_MACOSX size_t len = 0; sysctlbyname("hw.model", NULL, &len, NULL, 0); if (len) { UniquePtr model(new char[len]); // This string can be // MacBook%d,%d for a normal MacBook // MacBookPro%d,%d for a MacBook Pro // MacBookAir%d,%d for a Macbook Air sysctlbyname("hw.model", model.get(), &len, NULL, 0); char* substring = strstr(model.get(), "MacBook"); if (substring) { const size_t offset = strlen("MacBook"); if (strncmp(model.get() + offset, "Air", len - offset) || isdigit(model[offset + 1])) { return true; } } return false; } #endif return false; } bool AudioCallbackDriver::Init() { cubeb* cubebContext = CubebUtils::GetCubebContext(); if (!cubebContext) { NS_WARNING("Could not get cubeb context."); if (!mFromFallback) { CubebUtils::ReportCubebStreamInitFailure(true); } return false; } cubeb_stream_params output; cubeb_stream_params input; uint32_t latency_frames; bool firstStream = CubebUtils::GetFirstStream(); MOZ_ASSERT(!NS_IsMainThread(), "This is blocking and should never run on the main thread."); mSampleRate = output.rate = CubebUtils::PreferredSampleRate(); if (AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16) { output.format = CUBEB_SAMPLE_S16NE; } else { output.format = CUBEB_SAMPLE_FLOAT32NE; } // Query and set the number of channels this AudioCallbackDriver will use. mOutputChannels = std::max(1, mGraphImpl->AudioChannelCount()); mBuffer = AudioCallbackBufferWrapper(mOutputChannels); mScratchBuffer = SpillBuffer(mOutputChannels); output.channels = mOutputChannels; output.layout = CUBEB_LAYOUT_UNDEFINED; Maybe latencyPref = CubebUtils::GetCubebMSGLatencyInFrames(); if (latencyPref) { latency_frames = latencyPref.value(); } else { if (cubeb_get_min_latency(cubebContext, &output, &latency_frames) != CUBEB_OK) { NS_WARNING("Could not get minimal latency from cubeb."); } } // Macbook and MacBook air don't have enough CPU to run very low latency // MediaStreamGraphs, cap the minimal latency to 512 frames int this case. if (IsMacbookOrMacbookAir()) { latency_frames = std::max((uint32_t) 512, latency_frames); } input = output; input.channels = mInputChannels; input.layout = CUBEB_LAYOUT_UNDEFINED; #ifdef MOZ_WEBRTC if (mGraphImpl->mInputWanted) { StaticMutexAutoLock lock(AudioInputCubeb::Mutex()); uint32_t userChannels = 0; AudioInputCubeb::GetUserChannelCount(mGraphImpl->mInputDeviceID, userChannels); input.channels = mInputChannels = std::min(8, userChannels); } #endif cubeb_stream* stream = nullptr; CubebUtils::AudioDeviceID input_id = nullptr, output_id = nullptr; // We have to translate the deviceID values to cubeb devid's since those can be // freed whenever enumerate is called. { #ifdef MOZ_WEBRTC StaticMutexAutoLock lock(AudioInputCubeb::Mutex()); #endif if ((!mGraphImpl->mInputWanted #ifdef MOZ_WEBRTC || AudioInputCubeb::GetDeviceID(mGraphImpl->mInputDeviceID, input_id) #endif ) && (mGraphImpl->mOutputDeviceID == -1 // pass nullptr for ID for default output #ifdef MOZ_WEBRTC // XXX we should figure out how we would use a deviceID for output without webrtc. // Currently we don't set this though, so it's ok || AudioInputCubeb::GetDeviceID(mGraphImpl->mOutputDeviceID, output_id) #endif ) && // XXX Only pass input input if we have an input listener. Always // set up output because it's easier, and it will just get silence. // XXX Add support for adding/removing an input listener later. cubeb_stream_init(cubebContext, &stream, "AudioCallbackDriver", input_id, mGraphImpl->mInputWanted ? &input : nullptr, output_id, mGraphImpl->mOutputWanted ? &output : nullptr, latency_frames, DataCallback_s, StateCallback_s, this) == CUBEB_OK) { mAudioStream.own(stream); DebugOnly rv = cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale()); NS_WARNING_ASSERTION( rv == CUBEB_OK, "Could not set the audio stream volume in GraphDriver.cpp"); CubebUtils::ReportCubebBackendUsed(); } else { #ifdef MOZ_WEBRTC StaticMutexAutoUnlock unlock(AudioInputCubeb::Mutex()); #endif NS_WARNING("Could not create a cubeb stream for MediaStreamGraph, falling back to a SystemClockDriver"); // Only report failures when we're not coming from a driver that was // created itself as a fallback driver because of a previous audio driver // failure. if (!mFromFallback) { CubebUtils::ReportCubebStreamInitFailure(firstStream); } // Fall back to a driver using a normal thread. If needed, // the graph will try to re-open an audio stream later. MonitorAutoLock lock(GraphImpl()->GetMonitor()); SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl()); SetNextDriver(nextDriver); nextDriver->MarkAsFallback(); nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd); // We're not using SwitchAtNextIteration here, because there // won't be a next iteration if we don't restart things manually: // the audio stream just signaled that it's in error state. mGraphImpl->SetCurrentDriver(nextDriver); nextDriver->Start(); return true; } } SetMicrophoneActive(mGraphImpl->mInputWanted); cubeb_stream_register_device_changed_callback(mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s); if (!StartStream()) { LOG(LogLevel::Warning, ("AudioCallbackDriver couldn't start stream.")); return false; } LOG(LogLevel::Debug, ("AudioCallbackDriver started.")); return true; } void AudioCallbackDriver::Start() { if (mPreviousDriver) { if (mPreviousDriver->AsAudioCallbackDriver()) { LOG(LogLevel::Debug, ("Releasing audio driver off main thread.")); RefPtr releaseEvent = new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(); mPreviousDriver = nullptr; } else { LOG(LogLevel::Debug, ("Dropping driver reference for SystemClockDriver.")); MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver()); mFromFallback = mPreviousDriver->AsSystemClockDriver()->IsFallback(); mPreviousDriver = nullptr; } } LOG(LogLevel::Debug, ("Starting new audio driver off main thread, " "to ensure it runs after previous shutdown.")); RefPtr initEvent = new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT); - nsresult rv = initEvent->Dispatch(); - mScheduled = NS_SUCCEEDED(rv); + initEvent->Dispatch(); } bool AudioCallbackDriver::StartStream() { + mShouldFallbackIfError = true; if (cubeb_stream_start(mAudioStream) != CUBEB_OK) { NS_WARNING("Could not start cubeb stream for MSG."); return false; } { MonitorAutoLock mon(mGraphImpl->GetMonitor()); mStarted = true; } return true; } void AudioCallbackDriver::Stop() { if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) { NS_WARNING("Could not stop cubeb stream for MSG."); } } void AudioCallbackDriver::Revive() { // Note: only called on MainThread, without monitor // We know were weren't in a running state LOG(LogLevel::Debug, ("AudioCallbackDriver reviving.")); // If we were switching, switch now. Otherwise, start the audio thread again. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (NextDriver()) { RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); } else { LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl)); RefPtr initEvent = new AsyncCubebTask(this, AsyncCubebOperation::INIT); initEvent->Dispatch(); } } void AudioCallbackDriver::RemoveCallback() { if (mAddedMixer) { mGraphImpl->mMixer.RemoveCallback(this); mAddedMixer = false; } } void AudioCallbackDriver::WaitForNextIteration() { } void AudioCallbackDriver::WakeUp() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); mGraphImpl->GetMonitor().Notify(); } void AudioCallbackDriver::Shutdown() { LOG(LogLevel::Debug, ("Releasing audio driver off main thread (GraphDriver::Shutdown).")); RefPtr releaseEvent = new AsyncCubebTask(this, AsyncCubebOperation::SHUTDOWN); releaseEvent->Dispatch(NS_DISPATCH_SYNC); } #if defined(XP_WIN) void AudioCallbackDriver::ResetDefaultDevice() { if (cubeb_stream_reset_default_device(mAudioStream) != CUBEB_OK) { NS_WARNING("Could not reset cubeb stream to default output device."); } } #endif /* static */ long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser, const void* aInputBuffer, void* aOutputBuffer, long aFrames) { AudioCallbackDriver* driver = reinterpret_cast(aUser); return driver->DataCallback(static_cast(aInputBuffer), static_cast(aOutputBuffer), aFrames); } /* static */ void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void * aUser, cubeb_state aState) { AudioCallbackDriver* driver = reinterpret_cast(aUser); driver->StateCallback(aState); } /* static */ void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) { AudioCallbackDriver* driver = reinterpret_cast(aUser); driver->DeviceChangedCallback(); } bool AudioCallbackDriver::InCallback() { return mInCallback; } AudioCallbackDriver::AutoInCallback::AutoInCallback(AudioCallbackDriver* aDriver) : mDriver(aDriver) { mDriver->mInCallback = true; } AudioCallbackDriver::AutoInCallback::~AutoInCallback() { mDriver->mInCallback = false; } long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer, AudioDataValue* aOutputBuffer, long aFrames) { bool stillProcessing; // Don't add the callback until we're inited and ready if (!mAddedMixer) { mGraphImpl->mMixer.AddCallback(this); mAddedMixer = true; } #ifdef DEBUG // DebugOnly<> doesn't work here... it forces an initialization that will cause // mInCallback to be set back to false before we exit the statement. Do it by // hand instead. AutoInCallback aic(this); #endif GraphTime stateComputedTime = StateComputedTime(); if (stateComputedTime == 0) { MonitorAutoLock mon(mGraphImpl->GetMonitor()); // Because this function is called during cubeb_stream_init (to prefill the // audio buffers), it can be that we don't have a message here (because this // driver is the first one for this graph), and the graph would exit. Simply // return here until we have messages. if (!mGraphImpl->MessagesQueued()) { PodZero(aOutputBuffer, aFrames * mOutputChannels); return aFrames; } mGraphImpl->SwapMessageQueues(); } uint32_t durationMS = aFrames * 1000 / mSampleRate; // For now, simply average the duration with the previous // duration so there is some damping against sudden changes. if (!mIterationDurationMS) { mIterationDurationMS = durationMS; } else { mIterationDurationMS = (mIterationDurationMS*3) + durationMS; mIterationDurationMS /= 4; } // Process mic data if any/needed if (aInputBuffer) { if (mAudioInput) { // for this specific input-only or full-duplex stream mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer, static_cast(aFrames), mSampleRate, mInputChannels); } } mBuffer.SetBuffer(aOutputBuffer, aFrames); // fill part or all with leftover data from last iteration (since we // align to Audio blocks) mScratchBuffer.Empty(mBuffer); // if we totally filled the buffer (and mScratchBuffer isn't empty), // we don't need to run an iteration and if we do so we may overflow. if (mBuffer.Available()) { // State computed time is decided by the audio callback's buffer length. We // compute the iteration start and end from there, trying to keep the amount // of buffering in the graph constant. GraphTime nextStateComputedTime = mGraphImpl->RoundUpToNextAudioBlock(stateComputedTime + mBuffer.Available()); mIterationStart = mIterationEnd; // inGraph is the number of audio frames there is between the state time and // the current time, i.e. the maximum theoretical length of the interval we // could use as [mIterationStart; mIterationEnd]. GraphTime inGraph = stateComputedTime - mIterationStart; // We want the interval [mIterationStart; mIterationEnd] to be before the // interval [stateComputedTime; nextStateComputedTime]. We also want // the distance between these intervals to be roughly equivalent each time, to // ensure there is no clock drift between current time and state time. Since // we can't act on the state time because we have to fill the audio buffer, we // reclock the current time against the state time, here. mIterationEnd = mIterationStart + 0.8 * inGraph; LOG(LogLevel::Verbose, ("interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) " "(duration ticks: %ld)", (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime, (long)aFrames, (uint32_t)durationMS, (long)(nextStateComputedTime - stateComputedTime))); mCurrentTimeStamp = TimeStamp::Now(); if (stateComputedTime < mIterationEnd) { LOG(LogLevel::Warning, ("Media graph global underrun detected")); mIterationEnd = stateComputedTime; } stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); } else { LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch " "buffer, skipping iteration.")); stillProcessing = true; } mBuffer.BufferFilled(); // Callback any observers for the AEC speaker data. Note that one // (maybe) of these will be full-duplex, the others will get their input // data off separate cubeb callbacks. Take care with how stuff is // removed/added to this list and TSAN issues, but input and output will // use separate callback methods. mGraphImpl->NotifyOutputData(aOutputBuffer, static_cast(aFrames), mSampleRate, mOutputChannels); if (!stillProcessing) { + // About to hand over control of the graph. Do not start a new driver if + // StateCallback() receives an error for this stream while the main thread + // or another driver has control of the graph. + mShouldFallbackIfError = false; // Enter shutdown mode. The stable-state handler will detect this // and complete shutdown if the graph does not get restarted. mGraphImpl->SignalMainThreadCleanup(); return aFrames - 1; } bool switching = false; { MonitorAutoLock mon(mGraphImpl->GetMonitor()); switching = !!NextDriver(); } if (switching) { + mShouldFallbackIfError = false; // If the audio stream has not been started by the previous driver or // the graph itself, keep it alive. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (!IsStarted()) { return aFrames; } LOG(LogLevel::Debug, ("Switching to system driver.")); RemoveCallback(); NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(NextDriver()); NextDriver()->Start(); // Returning less than aFrames starts the draining and eventually stops the // audio thread. This function will never get called again. return aFrames - 1; } return aFrames; } void AudioCallbackDriver::StateCallback(cubeb_state aState) { LOG(LogLevel::Debug, ("AudioCallbackDriver State: %d", aState)); - if (aState == CUBEB_STATE_ERROR) { - if (!mAudioStream) { - // If we don't have an audio stream here, this means that the stream - // initialization has failed. A fallback on a SystemCallDriver will happen at - // the callsite of `cubeb_stream_init`. - return; - } - + if (aState == CUBEB_STATE_ERROR && mShouldFallbackIfError) { MonitorAutoLock lock(GraphImpl()->GetMonitor()); - - if (NextDriver() && NextDriver()->Scheduled()) { - // We are switching to another driver that has already been scheduled - // to be initialized and started. There's nothing for us to do here. - return; - } - // Fall back to a driver using a normal thread. If needed, // the graph will try to re-open an audio stream later. SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl()); SetNextDriver(nextDriver); RemoveCallback(); nextDriver->MarkAsFallback(); nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd); // We're not using SwitchAtNextIteration here, because there // won't be a next iteration if we don't restart things manually: // the audio stream just signaled that it's in error state. mGraphImpl->SetCurrentDriver(nextDriver); nextDriver->Start(); } } void AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) { uint32_t toWrite = mBuffer.Available(); if (!mBuffer.Available()) { NS_WARNING("DataCallback buffer full, expect frame drops."); } MOZ_ASSERT(mBuffer.Available() <= aFrames); mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available()); MOZ_ASSERT(mBuffer.Available() == 0, "Missing frames to fill audio callback's buffer."); DebugOnly written = mScratchBuffer.Fill(aMixedBuffer + toWrite * aChannels, aFrames - toWrite); NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames."); }; void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) { #ifdef XP_MACOSX cubeb_device* out; int rv; char name[128]; size_t length = sizeof(name); rv = sysctlbyname("hw.model", name, &length, NULL, 0); if (rv) { return; } if (!strncmp(name, "MacBookPro", 10)) { if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) { // Check if we are currently outputing sound on external speakers. if (!strcmp(out->output_name, "ispk")) { // Pan everything to the right speaker. if (aMicrophoneActive) { if (cubeb_stream_set_panning(mAudioStream, 1.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the right."); } } else { if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the center."); } } } else { if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) { NS_WARNING("Could not pan audio output to the center."); } } cubeb_stream_device_destroy(mAudioStream, out); } } #endif } void AudioCallbackDriver::DeviceChangedCallback() { // Tell the audio engine the device has changed, it might want to reset some // state. MonitorAutoLock mon(mGraphImpl->GetMonitor()); if (mAudioInput) { mAudioInput->DeviceChanged(); } #ifdef XP_MACOSX PanOutputIfNeeded(mMicrophoneActive); #endif } void AudioCallbackDriver::SetMicrophoneActive(bool aActive) { mMicrophoneActive = aActive; #ifdef XP_MACOSX PanOutputIfNeeded(mMicrophoneActive); #endif } uint32_t AudioCallbackDriver::IterationDuration() { // The real fix would be to have an API in cubeb to give us the number. Short // of that, we approximate it here. bug 1019507 return mIterationDurationMS; } bool AudioCallbackDriver::IsStarted() { mGraphImpl->GetMonitor().AssertCurrentThreadOwns(); return mStarted; } void AudioCallbackDriver::EnqueueStreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation) { MonitorAutoLock mon(mGraphImpl->GetMonitor()); mPromisesForOperation.AppendElement(StreamAndPromiseForOperation(aStream, aPromise, aOperation)); } void AudioCallbackDriver::CompleteAudioContextOperations(AsyncCubebOperation aOperation) { AutoTArray array; // We can't lock for the whole function because AudioContextOperationCompleted // will grab the monitor { MonitorAutoLock mon(GraphImpl()->GetMonitor()); array.SwapElements(mPromisesForOperation); } for (uint32_t i = 0; i < array.Length(); i++) { StreamAndPromiseForOperation& s = array[i]; if ((aOperation == AsyncCubebOperation::INIT && s.mOperation == dom::AudioContextOperation::Resume) || (aOperation == AsyncCubebOperation::SHUTDOWN && s.mOperation != dom::AudioContextOperation::Resume)) { GraphImpl()->AudioContextOperationCompleted(s.mStream, s.mPromise, s.mOperation); array.RemoveElementAt(i); i--; } } if (!array.IsEmpty()) { MonitorAutoLock mon(GraphImpl()->GetMonitor()); mPromisesForOperation.AppendElements(array); } } } // namespace mozilla // avoid redefined macro in unified build #undef LOG diff --git a/dom/media/GraphDriver.h b/dom/media/GraphDriver.h --- a/dom/media/GraphDriver.h +++ b/dom/media/GraphDriver.h @@ -1,593 +1,595 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef GRAPHDRIVER_H_ #define GRAPHDRIVER_H_ #include "nsAutoRef.h" #include "AudioBufferUtils.h" #include "AudioMixer.h" #include "AudioSegment.h" #include "SelfRef.h" #include "mozilla/Atomics.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/StaticPtr.h" #if defined(XP_WIN) #include "mozilla/audio/AudioNotificationReceiver.h" #endif struct cubeb_stream; template <> class nsAutoRefTraits : public nsPointerRefTraits { public: static void Release(cubeb_stream* aStream) { cubeb_stream_destroy(aStream); } }; namespace mozilla { /** * Assume we can run an iteration of the MediaStreamGraph loop in this much time * or less. * We try to run the control loop at this rate. */ static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10; /** * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by * this much. */ static const int SCHEDULE_SAFETY_MARGIN_MS = 10; /** * Try have this much audio buffered in streams and queued to the hardware. * The maximum delay to the end of the next control loop * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS. * There is no point in buffering more audio than this in a stream at any * given time (until we add processing). * This is not optimal yet. */ static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS; class MediaStreamGraphImpl; class AudioCallbackDriver; class OfflineClockDriver; class SystemClockDriver; /** * A driver is responsible for the scheduling of the processing, the thread * management, and give the different clocks to a MediaStreamGraph. This is an * abstract base class. A MediaStreamGraph can be driven by an * OfflineClockDriver, if the graph is offline, or a SystemClockDriver, if the * graph is real time. * A MediaStreamGraph holds an owning reference to its driver. * * The lifetime of drivers is a complicated affair. Here are the different * scenarii that can happen: * * Starting a MediaStreamGraph with an AudioCallbackDriver * - A new thread T is created, from the main thread. * - On this thread T, cubeb is initialized if needed, and a cubeb_stream is * created and started * - The thread T posts a message to the main thread to terminate itself. * - The graph runs off the audio thread * * Starting a MediaStreamGraph with a SystemClockDriver: * - A new thread T is created from the main thread. * - The graph runs off this thread. * * Switching from a SystemClockDriver to an AudioCallbackDriver: * - A new AudioCallabackDriver is created and initialized on the graph thread * - At the end of the MSG iteration, the SystemClockDriver transfers its timing * info and a reference to itself to the AudioCallbackDriver. It then starts * the AudioCallbackDriver. * - When the AudioCallbackDriver starts, it checks if it has been switched from * a SystemClockDriver, and if that is the case, sends a message to the main * thread to shut the SystemClockDriver thread down. * - The graph now runs off an audio callback * * Switching from an AudioCallbackDriver to a SystemClockDriver: * - A new SystemClockDriver is created, and set as mNextDriver. * - At the end of the MSG iteration, the AudioCallbackDriver transfers its * timing info and a reference to itself to the SystemClockDriver. A new * SystemClockDriver is started from the current audio thread. * - When starting, the SystemClockDriver checks if it has been switched from an * AudioCallbackDriver. If yes, it creates a new temporary thread to release * the cubeb_streams. This temporary thread closes the cubeb_stream, and * then dispatches a message to the main thread to be terminated. * - The graph now runs off a normal thread. * * Two drivers cannot run at the same time for the same graph. The thread safety * of the different attributes of drivers, and they access pattern is documented * next to the members themselves. * */ class GraphDriver { public: explicit GraphDriver(MediaStreamGraphImpl* aGraphImpl); NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GraphDriver); /* For real-time graphs, this waits until it's time to process more data. For * offline graphs, this is a no-op. */ virtual void WaitForNextIteration() = 0; /* Wakes up the graph if it is waiting. */ virtual void WakeUp() = 0; /* Start the graph, init the driver, start the thread. */ virtual void Start() = 0; /* Revive this driver, as more messages just arrived. */ virtual void Revive() = 0; /* Remove Mixer callbacks when switching */ virtual void RemoveCallback() = 0; /* Shutdown GraphDriver (synchronously) */ virtual void Shutdown() = 0; /* Rate at which the GraphDriver runs, in ms. This can either be user * controlled (because we are using a {System,Offline}ClockDriver, and decide * how often we want to wakeup/how much we want to process per iteration), or * it can be indirectly set by the latency of the audio backend, and the * number of buffers of this audio backend: say we have four buffers, and 40ms * latency, we will get a callback approximately every 10ms. */ virtual uint32_t IterationDuration() = 0; /* Return whether we are switching or not. */ bool Switching(); // Those are simply or setting the associated pointer, but assert that the // lock is held. GraphDriver* NextDriver(); GraphDriver* PreviousDriver(); void SetNextDriver(GraphDriver* aNextDriver); void SetPreviousDriver(GraphDriver* aPreviousDriver); - /* Return whether we have been scheduled to start. */ - bool Scheduled(); - /** * If we are running a real time graph, get the current time stamp to schedule * video frames. This has to be reimplemented by real time drivers. */ virtual TimeStamp GetCurrentTimeStamp() { return mCurrentTimeStamp; } GraphTime IterationEnd() { return mIterationEnd; } virtual AudioCallbackDriver* AsAudioCallbackDriver() { return nullptr; } virtual OfflineClockDriver* AsOfflineClockDriver() { return nullptr; } virtual SystemClockDriver* AsSystemClockDriver() { return nullptr; } /** * Tell the driver it has to stop and return the current time of the graph, so * another driver can start from the right point in time. */ virtual void SwitchAtNextIteration(GraphDriver* aDriver); /** * Set the time for a graph, on a driver. This is used so a new driver just * created can start at the right point in time. */ void SetGraphTime(GraphDriver* aPreviousDriver, GraphTime aLastSwitchNextIterationStart, GraphTime aLastSwitchNextIterationEnd); /** * Call this to indicate that another iteration of the control loop is * required on its regular schedule. The monitor must not be held. * This function has to be idempotent. */ void EnsureNextIteration(); /** * Same thing, but not locked. */ void EnsureNextIterationLocked(); MediaStreamGraphImpl* GraphImpl() { return mGraphImpl; } virtual bool OnThread() = 0; protected: GraphTime StateComputedTime() const; // Time of the start of this graph iteration. This must be accessed while // having the monitor. GraphTime mIterationStart; // Time of the end of this graph iteration. This must be accessed while having // the monitor. GraphTime mIterationEnd; // The MediaStreamGraphImpl that owns this driver. This has a lifetime longer // than the driver, and will never be null. Hence, it can be accesed without // monitor. MediaStreamGraphImpl* mGraphImpl; // This is used on the main thread (during initialization), and the graph // thread. No monitor needed because we know the graph thread does not run // during the initialization. TimeStamp mCurrentTimeStamp; // This is non-null only when this driver has recently switched from an other // driver, and has not cleaned it up yet (for example because the audio stream // is currently calling the callback during initialization). // // This is written to when changing driver, from the previous driver's thread, // or a thread created for the occasion. This is read each time we need to // check whether we're changing driver (in Switching()), from the graph // thread. // This must be accessed using the {Set,Get}PreviousDriver methods. RefPtr mPreviousDriver; // This is non-null only when this driver is going to switch to an other // driver at the end of this iteration. // This must be accessed using the {Set,Get}NextDriver methods. RefPtr mNextDriver; - // This is initially false, but set to true as soon the driver has been - // scheduled to start through GraphDriver::Start(). - bool mScheduled; virtual ~GraphDriver() { } }; class MediaStreamGraphInitThreadRunnable; /** * This class is a driver that manages its own thread. */ class ThreadedDriver : public GraphDriver { public: explicit ThreadedDriver(MediaStreamGraphImpl* aGraphImpl); virtual ~ThreadedDriver(); void Start() override; void Revive() override; void RemoveCallback() override; void Shutdown() override; /** * Runs main control loop on the graph thread. Normally a single invocation * of this runs for the entire lifetime of the graph thread. */ void RunThread(); friend class MediaStreamGraphInitThreadRunnable; uint32_t IterationDuration() override { return MEDIA_GRAPH_TARGET_PERIOD_MS; } bool OnThread() override { return !mThread || mThread->EventTarget()->IsOnCurrentThread(); } /* When the graph wakes up to do an iteration, implementations return the * range of time that will be processed. This is called only once per * iteration; it may determine the interval from state in a previous * call. */ virtual MediaTime GetIntervalForIteration() = 0; protected: nsCOMPtr mThread; }; /** * A SystemClockDriver drives a MediaStreamGraph using a system clock, and waits * using a monitor, between each iteration. */ class SystemClockDriver : public ThreadedDriver { public: explicit SystemClockDriver(MediaStreamGraphImpl* aGraphImpl); virtual ~SystemClockDriver(); MediaTime GetIntervalForIteration() override; void WaitForNextIteration() override; void WakeUp() override; void MarkAsFallback(); bool IsFallback(); SystemClockDriver* AsSystemClockDriver() override { return this; } private: // Those are only modified (after initialization) on the graph thread. The // graph thread does not run during the initialization. TimeStamp mInitialTimeStamp; TimeStamp mLastTimeStamp; // This enum specifies the wait state of the driver. enum WaitState { // RunThread() is running normally WAITSTATE_RUNNING, // RunThread() is paused waiting for its next iteration, which will // happen soon WAITSTATE_WAITING_FOR_NEXT_ITERATION, // RunThread() is paused indefinitely waiting for something to change WAITSTATE_WAITING_INDEFINITELY, // Something has signaled RunThread() to wake up immediately, // but it hasn't done so yet WAITSTATE_WAKING_UP }; // This must be access with the monitor. WaitState mWaitState; // This is true if this SystemClockDriver runs the graph because we could not // open an audio stream. bool mIsFallback; }; /** * An OfflineClockDriver runs the graph as fast as possible, without waiting * between iteration. */ class OfflineClockDriver : public ThreadedDriver { public: OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice); virtual ~OfflineClockDriver(); MediaTime GetIntervalForIteration() override; void WaitForNextIteration() override; void WakeUp() override; TimeStamp GetCurrentTimeStamp() override; OfflineClockDriver* AsOfflineClockDriver() override { return this; } private: // Time, in GraphTime, for each iteration GraphTime mSlice; }; struct StreamAndPromiseForOperation { StreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation); RefPtr mStream; void* mPromise; dom::AudioContextOperation mOperation; }; enum AsyncCubebOperation { INIT, SHUTDOWN }; /** * This is a graph driver that is based on callback functions called by the * audio api. This ensures minimal audio latency, because it means there is no * buffering happening: the audio is generated inside the callback. * * This design is less flexible than running our own thread: * - We have no control over the thread: * - It cannot block, and it has to run for a shorter amount of time than the * buffer it is going to fill, or an under-run is going to occur (short burst * of silence in the final audio output). * - We can't know for sure when the callback function is going to be called * (although we compute an estimation so we can schedule video frames) * - Creating and shutting the thread down is a blocking operation, that can * take _seconds_ in some cases (because IPC has to be set up, and * sometimes hardware components are involved and need to be warmed up) * - We have no control on how much audio we generate, we have to return exactly * the number of frames asked for by the callback. Since for the Web Audio * API, we have to do block processing at 128 frames per block, we need to * keep a little spill buffer to store the extra frames. */ class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver #if defined(XP_WIN) , public audio::DeviceChangeListener #endif { public: explicit AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl); virtual ~AudioCallbackDriver(); void Start() override; void Revive() override; void RemoveCallback() override; void WaitForNextIteration() override; void WakeUp() override; void Shutdown() override; #if defined(XP_WIN) void ResetDefaultDevice() override; #endif /* Static wrapper function cubeb calls back. */ static long DataCallback_s(cubeb_stream * aStream, void * aUser, const void * aInputBuffer, void * aOutputBuffer, long aFrames); static void StateCallback_s(cubeb_stream* aStream, void * aUser, cubeb_state aState); static void DeviceChangedCallback_s(void * aUser); /* This function is called by the underlying audio backend when a refill is * needed. This is what drives the whole graph when it is used to output * audio. If the return value is exactly aFrames, this function will get * called again. If it is less than aFrames, the stream will go in draining * mode, and this function will not be called again. */ long DataCallback(const AudioDataValue* aInputBuffer, AudioDataValue* aOutputBuffer, long aFrames); /* This function is called by the underlying audio backend, but is only used * for informational purposes at the moment. */ void StateCallback(cubeb_state aState); /* This is an approximation of the number of millisecond there are between two * iterations of the graph. */ uint32_t IterationDuration() override; /* This function gets called when the graph has produced the audio frames for * this iteration. */ void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) override; // These are invoked on the MSG thread (we don't call this if not LIFECYCLE_RUNNING) virtual void SetInputListener(AudioDataListener *aListener) { MOZ_ASSERT(OnThread()); mAudioInput = aListener; } // XXX do we need the param? probably no virtual void RemoveInputListener(AudioDataListener *aListener) { MOZ_ASSERT(OnThread()); mAudioInput = nullptr; } AudioCallbackDriver* AsAudioCallbackDriver() override { return this; } uint32_t OutputChannelCount() { MOZ_ASSERT(mOutputChannels != 0 && mOutputChannels <= 8); return mOutputChannels; } /* Enqueue a promise that is going to be resolved when a specific operation * occurs on the cubeb stream. */ void EnqueueStreamAndPromiseForOperation(MediaStream* aStream, void* aPromise, dom::AudioContextOperation aOperation); /** * Whether the audio callback is processing. This is for asserting only. */ bool InCallback(); bool OnThread() override { return !mStarted || InCallback(); } /* Whether the underlying cubeb stream has been started. See comment for * mStarted for details. */ bool IsStarted(); /* Tell the driver whether this process is using a microphone or not. This is * thread safe. */ void SetMicrophoneActive(bool aActive); void CompleteAudioContextOperations(AsyncCubebOperation aOperation); /* Fetch, or create a shared thread pool with up to one thread for * AsyncCubebTask. */ SharedThreadPool* GetInitShutdownThread(); private: /** * On certain MacBookPro, the microphone is located near the left speaker. * We need to pan the sound output to the right speaker if we are using the * mic and the built-in speaker, or we will have terrible echo. */ void PanOutputIfNeeded(bool aMicrophoneActive); /** * This is called when the output device used by the cubeb stream changes. */ void DeviceChangedCallback(); /* Start the cubeb stream */ bool StartStream(); friend class AsyncCubebTask; bool Init(); void Stop(); /* MediaStreamGraphs are always down/up mixed to output channels. */ uint32_t mOutputChannels; /* The size of this buffer comes from the fact that some audio backends can * call back with a number of frames lower than one block (128 frames), so we * need to keep at most two block in the SpillBuffer, because we always round * up to block boundaries during an iteration. * This is only ever accessed on the audio callback thread. */ SpillBuffer mScratchBuffer; /* Wrapper to ensure we write exactly the number of frames we need in the * audio buffer cubeb passes us. This is only ever accessed on the audio * callback thread. */ AudioCallbackBufferWrapper mBuffer; /* cubeb stream for this graph. This is guaranteed to be non-null after Init() * has been called, and is synchronized internaly. */ nsAutoRef mAudioStream; /* The sample rate for the aforementionned cubeb stream. This is set on * initialization and can be read safely afterwards. */ uint32_t mSampleRate; /* The number of input channels from cubeb. Should be set before opening cubeb * and then be static. */ uint32_t mInputChannels; /* Approximation of the time between two callbacks. This is used to schedule * video frames. This is in milliseconds. Only even used (after * inizatialization) on the audio callback thread. */ uint32_t mIterationDurationMS; /* cubeb_stream_init calls the audio callback to prefill the buffers. The * previous driver has to be kept alive until the audio stream has been * started, because it is responsible to call cubeb_stream_start, so we delay * the cleanup of the previous driver until it has started the audio stream. * Otherwise, there is a race where we kill the previous driver thread * between cubeb_stream_init and cubeb_stream_start, * and callbacks after the prefill never get called. * This is written on the previous driver's thread (if switching) or main * thread (if this driver is the first one). * This is read on previous driver's thread (during callbacks from * cubeb_stream_init) and the audio thread (when switching away from this * driver back to a SystemClockDriver). * This is synchronized by the Graph's monitor. * */ bool mStarted; /* Listener for mic input, if any. */ RefPtr mAudioInput; struct AutoInCallback { explicit AutoInCallback(AudioCallbackDriver* aDriver); ~AutoInCallback(); AudioCallbackDriver* mDriver; }; /* Shared thread pool with up to one thread for off-main-thread * initialization and shutdown of the audio stream via AsyncCubebTask. */ RefPtr mInitShutdownThread; /* This must be accessed with the graph monitor held. */ AutoTArray mPromisesForOperation; /* Used to queue us to add the mixer callback on first run. */ bool mAddedMixer; /* This is atomic and is set by the audio callback thread. It can be read by * any thread safely. */ Atomic mInCallback; /** * True if microphone is being used by this process. This is synchronized by * the graph's monitor. */ Atomic mMicrophoneActive; + /* Indication of whether a fallback SystemClockDriver should be started if + * StateCallback() receives an error. No mutex need be held during access. + * The transition to true happens before cubeb_stream_start() is called. + * After transitioning to false on the last DataCallback(), the stream is + * not accessed from another thread until the graph thread either signals + * main thread cleanup or dispatches an event to switch to another + * driver. */ + bool mShouldFallbackIfError; /* True if this driver was created from a driver created because of a previous * AudioCallbackDriver failure. */ bool mFromFallback; }; class AsyncCubebTask : public Runnable { public: AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation); nsresult Dispatch(uint32_t aFlags = NS_DISPATCH_NORMAL) { SharedThreadPool* threadPool = mDriver->GetInitShutdownThread(); if (!threadPool) { return NS_ERROR_FAILURE; } return threadPool->Dispatch(this, aFlags); } protected: virtual ~AsyncCubebTask(); private: NS_IMETHOD Run() override final; RefPtr mDriver; AsyncCubebOperation mOperation; RefPtr mShutdownGrip; }; } // namespace mozilla #endif // GRAPHDRIVER_H_