VIDCS-2175: Should mitigate against publisher failures via permissions, hardware, or network failures by automatically retrying. (no ghost viewers, if retrying doesn’t work we fail))

frontend/src/Context/PreviewPublisherProvider/usePreviewPublisher/usePreviewPublisher.spec.tsx

@@ -0,0 +1,146 @@
+import { act, cleanup, renderHook } from '@testing-library/react';
+import { afterAll, afterEach, beforeEach, describe, expect, it, Mock, vi } from 'vitest';
+import { initPublisher, Publisher } from '@vonage/client-sdk-video';
+import EventEmitter from 'events';
+import usePreviewPublisher from './usePreviewPublisher';
+import { UserContextType } from '../../user';
+import useUserContext from '../../../hooks/useUserContext';
+import usePermissions, { PermissionsHookType } from '../../../hooks/usePermissions';
+import useDevices from '../../../hooks/useDevices';
+import { AllMediaDevices } from '../../../types';
+import {
+  allMediaDevices,
+  defaultAudioDevice,
+  defaultVideoDevice,
+} from '../../../utils/mockData/device';
+import { DEVICE_ACCESS_STATUS } from '../../../utils/constants';
+
+vi.mock('@vonage/client-sdk-video');
+vi.mock('../../../hooks/useUserContext.tsx');
+vi.mock('../../../hooks/usePermissions.tsx');
+vi.mock('../../../hooks/useDevices.tsx');
+
+const mockUseUserContext = useUserContext as Mock<[], UserContextType>;
+const mockUsePermissions = usePermissions as Mock<[], PermissionsHookType>;
+const mockUseDevices = useDevices as Mock<
+  [],
+  { allMediaDevices: AllMediaDevices; getAllMediaDevices: () => void }
+>;
+
+const defaultSettings = {
+  publishAudio: false,
+  publishVideo: false,
+  name: '',
+  blur: false,
+  noiseSuppression: true,
+};
+const mockUserContextWithDefaultSettings = {
+  user: {
+    defaultSettings,
+  },
+} as UserContextType;
+
+describe('usePreviewPublisher', () => {
+  const mockPublisher = Object.assign(new EventEmitter(), {
+    getAudioSource: () => defaultAudioDevice,
+    getVideoSource: () => defaultVideoDevice,
+  }) as unknown as Publisher;
+  const mockedInitPublisher = vi.fn();
+  const consoleErrorSpy = vi.spyOn(console, 'error');
+  const mockSetAccessStatus = vi.fn();
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+
+    mockUseUserContext.mockImplementation(() => mockUserContextWithDefaultSettings);
+    (initPublisher as Mock).mockImplementation(mockedInitPublisher);
+    mockUseDevices.mockReturnValue({
+      getAllMediaDevices: vi.fn(),
+      allMediaDevices,
+    });
+    mockUsePermissions.mockReturnValue({
+      accessStatus: DEVICE_ACCESS_STATUS.PENDING,
+      setAccessStatus: mockSetAccessStatus,
+    });
+  });
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  describe('initLocalPublisher', () => {
+    it('should call initPublisher', () => {
+      mockedInitPublisher.mockReturnValue(mockPublisher);
+      (initPublisher as Mock).mockImplementation(mockedInitPublisher);
+      const { result } = renderHook(() => usePreviewPublisher());
+      result.current.initLocalPublisher();
+
+      expect(mockedInitPublisher).toHaveBeenCalled();
+    });
+
+    it('should log access denied errors', () => {
+      (initPublisher as Mock).mockImplementation((_, _args, callback) => {
+        const error = new Error(
+          "It hit me pretty hard, how there's no kind of sad in this world that will stop it turning."
+        );
+        error.name = 'OT_USER_MEDIA_ACCESS_DENIED';
+        callback(error);
+      });
+
+      const { result } = renderHook(() => usePreviewPublisher());
+      act(() => {
+        result.current.initLocalPublisher();
+      });
+      expect(consoleErrorSpy).toHaveBeenCalled();
+    });
+  });
+
+  describe('on accessDenied', () => {
+    const nativePermissions = global.navigator.permissions;
+    const mockQuery = vi.fn();
+    let mockedPermissionStatus: { onchange: null | (() => void); status: string };
+
+    beforeEach(() => {
+      mockedPermissionStatus = {
+        onchange: null,
+        status: 'prompt',
+      };
+      mockQuery.mockResolvedValue(mockedPermissionStatus);
+
+      Object.defineProperty(global.navigator, 'permissions', {
+        writable: true,
+        value: {
+          query: mockQuery,
+        },
+      });
+    });
+
+    afterAll(() => {
+      Object.defineProperty(global.navigator, 'permissions', {
+        writable: true,
+        value: nativePermissions,
+      });
+    });
+
+    it('calls setAccessStatus', async () => {
+      mockedInitPublisher.mockReturnValue(mockPublisher);
+      (initPublisher as Mock).mockImplementation(mockedInitPublisher);
+
+      const { result } = renderHook(() => usePreviewPublisher());
+
+      act(() => {
+        result.current.initLocalPublisher();
+      });
+      expect(result.current.accessStatus).toBe(DEVICE_ACCESS_STATUS.PENDING);
+
+      act(() => {
+        // @ts-expect-error We simulate user denying microphone permissions in a browser.
+        mockPublisher.emit('accessDenied', {
+          message: 'microphone permission denied during the call',
+        });
+      });
+
+      expect(mockSetAccessStatus).toBeCalledWith(DEVICE_ACCESS_STATUS.REJECTED);
+    });
+  });
+});

frontend/src/Context/PreviewPublisherProvider/usePreviewPublisher/usePreviewPublisher.tsx

@@ -6,6 +6,7 @@ import usePermissions from '../../../hooks/usePermissions';
 import useUserContext from '../../../hooks/useUserContext';
 import { DEVICE_ACCESS_STATUS } from '../../../utils/constants';
 import { UserType } from '../../user';
+import { AccessDeniedEvent } from '../../PublisherProvider/usePublisher/usePublisher';
 
 type PublisherVideoElementCreatedEvent = Event<'videoElementCreated', Publisher> & {
   element: HTMLVideoElement | HTMLObjectElement;
@@ -150,15 +151,28 @@ const usePreviewPublisher = (): PreviewPublisherContextType => {
   /**
    * Handle device permissions denial
    * used to inform the user they need to give permissions to devices to access the call
+   * after a user grants permissions to the denied device, trigger a reload.
    * @returns {void}
    */
-  const handleAccessDenied = useCallback(() => {
-    console.log('access denied');
+  const handleAccessDenied = useCallback(
+    (event: AccessDeniedEvent) => {
+      const deviceDeniedAccess =
+        event.message?.match(/^(\w*)/)?.[0] === 'microphone' ? 'microphone' : 'camera';
 
-    setAccessStatus(DEVICE_ACCESS_STATUS.REJECTED);
+      setAccessStatus(DEVICE_ACCESS_STATUS.REJECTED);
 
-    publisherRef.current = null;
-  }, [setAccessStatus]);
+      // @ts-expect-error The camera and microphone permissions are supported on all major browsers.
+      window.navigator.permissions.query({ name: deviceDeniedAccess }).then((permissionStatus) => {
+        // eslint-disable-next-line no-param-reassign
+        permissionStatus.onchange = () => {
+          if (permissionStatus.state === 'granted') {
+            setAccessStatus(DEVICE_ACCESS_STATUS.ACCESS_CHANGED);
+          }
+        };
+      });
+    },
+    [setAccessStatus]
+  );
 
   const handleVideoElementCreated = (event: PublisherVideoElementCreatedEvent) => {
     setPublisherVideoElement(event.element);
@@ -172,7 +186,10 @@ const usePreviewPublisher = (): PreviewPublisherContextType => {
   }, []);
 
   const addPublisherListeners = useCallback(
-    (publisher: Publisher) => {
+    (publisher: Publisher | null) => {
+      if (!publisher) {
+        return;
+      }
       publisher.on('destroyed', handleDestroyed);
       publisher.on('accessDenied', handleAccessDenied);
       publisher.on('videoElementCreated', handleVideoElementCreated);

frontend/src/Context/PublisherProvider/usePublisher/usePublisher.spec.tsx

@@ -1,12 +1,13 @@
 import { beforeEach, describe, it, expect, vi, Mock, afterAll, Mocked } from 'vitest';
 import { act, renderHook } from '@testing-library/react';
-import { initPublisher, Session, Stream } from '@vonage/client-sdk-video';
+import { initPublisher, Publisher, Session, Stream } from '@vonage/client-sdk-video';
 import EventEmitter from 'events';
 import usePublisher from './usePublisher';
 import useUserContext from '../../../hooks/useUserContext';
 import { UserContextType } from '../../user';
 import useSessionContext from '../../../hooks/useSessionContext';
 import { SessionContextType } from '../../SessionProvider/session';
+import { PUBLISHING_BLOCKED_CAPTION } from '../../../utils/constants';
 
 vi.mock('@vonage/client-sdk-video');
 vi.mock('../../../hooks/useUserContext.tsx');
@@ -33,7 +34,10 @@ const mockStream = {
 } as unknown as Stream;
 
 describe('usePublisher', () => {
-  const mockPublisher = new EventEmitter();
+  const destroySpy = vi.fn();
+  const mockPublisher = Object.assign(new EventEmitter(), {
+    destroy: destroySpy,
+  }) as unknown as Publisher;
   let sessionContext: SessionContextType;
   let sessionMock: Mocked<Session>;
   const mockedInitPublisher = vi.fn();
@@ -142,6 +146,7 @@ describe('usePublisher', () => {
 
       act(() => {
         result.current.initializeLocalPublisher();
+        // @ts-expect-error We simulate the publisher stream being created.
         mockPublisher.emit('streamCreated', { stream: mockStream });
       });
       expect(initPublisher).toHaveBeenCalledOnce();
@@ -174,8 +179,38 @@ describe('usePublisher', () => {
         await result.current.publish();
       });
 
-      expect(result.current.isPublishingError).toEqual(true);
+      const publishingBlockedError = {
+        header: 'Difficulties joining room',
+        caption: PUBLISHING_BLOCKED_CAPTION,
+      };
+      expect(result.current.publishingError).toEqual(publishingBlockedError);
       expect(mockedSessionPublish).toHaveBeenCalledTimes(2);
     });
   });
+
+  it('should set publishingError and destroy publisher when receiving an accessDenied event', () => {
+    (initPublisher as Mock).mockImplementation(() => mockPublisher);
+    const { result } = renderHook(() => usePublisher());
+
+    act(() => {
+      result.current.initializeLocalPublisher();
+    });
+
+    expect(result.current.publishingError).toBeNull();
+
+    act(() => {
+      // @ts-expect-error We simulate user denying microphone permissions in a browser.
+      mockPublisher.emit('accessDenied', {
+        message: 'microphone permission denied during the call',
+      });
+    });
+
+    expect(result.current.publishingError).toEqual({
+      header: 'Camera access is denied',
+      caption:
+        "It seems your browser is blocked from accessing your camera. Reset the permission state through your browser's UI.",
+    });
+    expect(destroySpy).toHaveBeenCalled();
+    expect(result.current.publisher).toBeNull();
+  });
 });

frontend/src/Context/PublisherProvider/usePublisher/usePublisher.tsx

@@ -9,6 +9,11 @@ import OT, {
 import usePublisherQuality, { NetworkQuality } from '../usePublisherQuality/usePublisherQuality';
 import usePublisherOptions from '../usePublisherOptions';
 import useSessionContext from '../../../hooks/useSessionContext';
+import { PUBLISHING_BLOCKED_CAPTION } from '../../../utils/constants';
+import getAccessDeniedError, {
+  PublishingError,
+  PublishingErrorType,
+} from '../../../utils/getAccessDeniedError/getAccessDeniedError';
 
 type PublisherStreamCreatedEvent = Event<'streamCreated', Publisher> & {
   stream: Stream;
@@ -18,12 +23,21 @@ type PublisherVideoElementCreatedEvent = Event<'videoElementCreated', Publisher>
   element: HTMLVideoElement | HTMLObjectElement;
 };
 
+type DeviceAccessStatus = {
+  microphone: boolean | undefined;
+  camera: boolean | undefined;
+};
+
+export type AccessDeniedEvent = Event<'accessDenied', Publisher> & {
+  message?: string;
+};
+
 export type PublisherContextType = {
   initializeLocalPublisher: () => void;
   isAudioEnabled: boolean;
   isForceMuted: boolean;
   isPublishing: boolean;
-  isPublishingError: boolean;
+  publishingError: PublishingError;
   isVideoEnabled: boolean;
   publish: () => Promise<void>;
   publisher: Publisher | null;
@@ -41,7 +55,7 @@ export type PublisherContextType = {
  * @property {() => void} initializeLocalPublisher - Method to initialize publisher
  * @property {boolean} isAudioEnabled - React state boolean showing if audio is enabled
  * @property {boolean} isPublishing - React state boolean showing if we are publishing
- * @property {boolean} isPublishingError - React state boolean showing if we are unable to publish to the session.
+ * @property {boolean} publishingError - React state showing any errors thrown while attempting to publish.
  * @property {boolean} isVideoEnabled - React state boolean showing if camera is on
  * @property {boolean} isForceMuted - React state boolean showing if the end user was force muted
  * @property {() => Promise<void>} publish - Method to publish to session
@@ -67,10 +81,30 @@ const usePublisher = (): PublisherContextType => {
   const [isAudioEnabled, setIsAudioEnabled] = useState(!!publisherOptions.publishAudio);
   const [stream, setStream] = useState<Stream | null>();
   const [isPublishingToSession, setIsPublishingToSession] = useState(false);
-  const [isPublishingError, setIsPublishingError] = useState(false);
+  const [publishingError, setPublishingError] = useState<PublishingError>(null);
   const mSession = useSessionContext();
+  const [deviceAccess, setDeviceAccess] = useState<DeviceAccessStatus>({
+    microphone: undefined,
+    camera: undefined,
+  });
   let publishAttempt: number = 0;
 
+  // If we do not have audio input or video input access, we cannot publish.
+  useEffect(() => {
+    if (deviceAccess?.microphone === false || deviceAccess.camera === false) {
+      const device = deviceAccess.camera ? 'Microphone' : 'Camera';
+      const accessDeniedError = getAccessDeniedError(device);
+      setPublishingError(accessDeniedError);
+    }
+  }, [deviceAccess]);
+
+  const handleAccessAllowed = () => {
+    setDeviceAccess({
+      microphone: true,
+      camera: true,
+    });
+  };
+
   const handleDestroyed = () => {
     publisherRef.current = null;
   };
@@ -89,7 +123,15 @@ const usePublisher = (): PublisherContextType => {
     publisherRef.current = null;
   };
 
-  const handleAccessDenied = () => {
+  const handleAccessDenied = (event: AccessDeniedEvent) => {
+    // We check the first word of the message to see if the microphone or camera was denied access.
+    const deviceDeniedAccess =
+      event.message?.match(/^(\w*)/)?.[0] === 'microphone' ? 'microphone' : 'camera';
+    setDeviceAccess((prev) => ({
+      ...prev,
+      [deviceDeniedAccess]: false,
+    }));
+
     if (publisherRef.current) {
       publisherRef.current.destroy();
     }
@@ -129,6 +171,7 @@ const usePublisher = (): PublisherContextType => {
     publisher.on('accessDenied', handleAccessDenied);
     publisher.on('videoElementCreated', handleVideoElementCreated);
     publisher.on('muteForced', handleMuteForced);
+    publisher.on('accessAllowed', handleAccessAllowed);
   };
 
   /**
@@ -156,7 +199,11 @@ const usePublisher = (): PublisherContextType => {
     publishAttempt += 1;
 
     if (publishAttempt === 3) {
-      setIsPublishingError(true);
+      const publishingBlocked: PublishingErrorType = {
+        header: 'Difficulties joining room',
+        caption: PUBLISHING_BLOCKED_CAPTION,
+      };
+      setPublishingError(publishingBlocked);
       setIsPublishingToSession(false);
       return true;
     }
@@ -247,7 +294,7 @@ const usePublisher = (): PublisherContextType => {
     isAudioEnabled,
     isForceMuted,
     isPublishing,
-    isPublishingError,
+    publishingError,
     isVideoEnabled,
     publish,
     publisher: publisherRef.current,