Skip to content

Commit

Permalink
Sync audio fragment loading with main discontinuity and time
Browse files Browse the repository at this point in the history
  • Loading branch information
@robwalch
robwalch committed Jun 12, 2024
1 parent 278e0e8 commit 20f50af
Show file tree
Hide file tree
Showing 19 changed files with 399 additions and 309 deletions.
6 changes: 3 additions & 3 deletions api-extractor/report/hls.js.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1429,7 +1429,7 @@ export class Fragment extends BaseSegment {
// (undocumented)
get end(): number;
// (undocumented)
endDTS: number;
endDTS?: number;
// (undocumented)
endList?: boolean;
// (undocumented)
Expand Down Expand Up @@ -1469,7 +1469,7 @@ export class Fragment extends BaseSegment {
// (undocumented)
start: number;
// (undocumented)
startDTS: number;
startDTS?: number;
// (undocumented)
startPTS?: number;
// (undocumented)
Expand Down Expand Up @@ -1976,7 +1976,7 @@ export interface InitPTSFoundData {
// (undocumented)
frag: MediaFragment;
// (undocumented)
id: string;
id: PlaylistLevelType;
// (undocumented)
initPTS: number;
// (undocumented)
Expand Down
168 changes: 85 additions & 83 deletions src/controller/audio-stream-controller.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import BaseStreamController, { State } from './base-stream-controller';
import { Events } from '../events';
import { Bufferable, BufferHelper } from '../utils/buffer-helper';
import { FragmentState } from './fragment-tracker';
import { Level } from '../types/level';
import { PlaylistContextType, PlaylistLevelType } from '../types/loader';
Expand All @@ -13,7 +12,7 @@ import {
import ChunkCache from '../demux/chunk-cache';
import TransmuxerInterface from '../demux/transmuxer-interface';
import { ChunkMetadata } from '../types/transmuxer';
import { fragmentWithinToleranceTest } from './fragment-finders';
import { findFragWithCC, findNearestWithCC } from './fragment-finders';
import { alignMediaPlaylistByPDT } from '../utils/discontinuities';
import { mediaAttributesIdentical } from '../utils/media-option-attributes';
import { ErrorDetails } from '../errors';
Expand All @@ -39,6 +38,8 @@ import type {
FragBufferedData,
ErrorData,
BufferFlushingData,
BufferCodecsData,
FragLoadingData,
} from '../types/events';
import type { MediaPlaylist } from '../types/media-playlist';

Expand All @@ -55,9 +56,8 @@ class AudioStreamController
extends BaseStreamController
implements NetworkComponentAPI
{
private videoBuffer: Bufferable | null = null;
private videoTrackCC: number = -1;
private waitingVideoCC: number = -1;
private videoAnchor: MediaFragment | null = null;
private mainFragLoading: FragLoadingData | null = null;
private bufferedTrack: MediaPlaylist | null = null;
private switchingTrack: MediaPlaylist | null = null;
private trackId: number = -1;
Expand Down Expand Up @@ -102,6 +102,7 @@ class AudioStreamController
hls.on(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
hls.on(Events.BUFFER_FLUSHED, this.onBufferFlushed, this);
hls.on(Events.INIT_PTS_FOUND, this.onInitPtsFound, this);
hls.on(Events.FRAG_LOADING, this.onFragLoading, this);
hls.on(Events.FRAG_BUFFERED, this.onFragBuffered, this);
}

Expand All @@ -120,6 +121,7 @@ class AudioStreamController
hls.off(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
hls.off(Events.BUFFER_FLUSHED, this.onBufferFlushed, this);
hls.off(Events.INIT_PTS_FOUND, this.onInitPtsFound, this);
hls.on(Events.FRAG_LOADING, this.onFragLoading, this);
hls.off(Events.FRAG_BUFFERED, this.onFragBuffered, this);
}

Expand All @@ -130,20 +132,44 @@ class AudioStreamController
) {
// Always update the new INIT PTS
// Can change due level switch
if (id === 'main') {
if (id === PlaylistLevelType.MAIN) {
const cc = frag.cc;
this.initPTS[frag.cc] = { baseTime: initPTS, timescale };
const inFlightFrag = this.fragCurrent;
this.initPTS[cc] = { baseTime: initPTS, timescale };
this.log(
`InitPTS for cc: ${cc} found from main: ${initPTS}/${timescale}`,
);
this.videoTrackCC = cc;
this.videoAnchor = frag;
// If we are waiting, tick immediately to unblock audio fragment transmuxing
if (this.state === State.WAITING_INIT_PTS) {
const waitingData = this.waitingData;
if (!waitingData || waitingData.frag.cc !== cc) {
this.nextLoadPosition = this.findSyncFrag(frag).start;
}
this.tick();
} else if (
!this.loadedmetadata &&
inFlightFrag &&
inFlightFrag.cc !== cc
) {
this.startFragRequested = false;
this.nextLoadPosition = this.findSyncFrag(frag).start;
inFlightFrag.abortRequests();
this.resetLoadingState();
}
}
}

private findSyncFrag(mainFrag: MediaFragment): MediaFragment {
const trackDetails = this.getLevelDetails();
const cc = mainFrag.cc;
return (
findNearestWithCC(trackDetails, cc, mainFrag) ||
(trackDetails && findFragWithCC(trackDetails.fragments, cc)) ||
mainFrag
);
}

startLoad(startPosition: number) {
if (!this.levels) {
this.startPosition = startPosition;
Expand Down Expand Up @@ -206,9 +232,9 @@ class AudioStreamController
const waitingData = this.waitingData;
if (waitingData) {
const { frag, part, cache, complete } = waitingData;
const videoAnchor = this.videoAnchor;
if (this.initPTS[frag.cc] !== undefined) {
this.waitingData = null;
this.waitingVideoCC = -1;
this.state = State.FRAG_LOADING;
const payload = cache.flush();
const data: FragLoadedData = {
Expand All @@ -221,33 +247,15 @@ class AudioStreamController
if (complete) {
super._handleFragmentLoadComplete(data);
}
} else if (this.videoTrackCC !== this.waitingVideoCC) {
} else if (videoAnchor && videoAnchor.cc !== waitingData.frag.cc) {
// Drop waiting fragment if videoTrackCC has changed since waitingFragment was set and initPTS was not found
this.log(
`Waiting fragment cc (${frag.cc}) cancelled because video is at cc ${this.videoTrackCC}`,
`Waiting fragment cc (${frag.cc}) cancelled because video is at cc ${videoAnchor.cc}`,
);
this.nextLoadPosition = this.findSyncFrag(videoAnchor).start;
this.clearWaitingFragment();
} else {
// Drop waiting fragment if an earlier fragment is needed
const pos = this.getLoadPosition();
const bufferInfo = BufferHelper.bufferInfo(
this.mediaBuffer,
pos,
this.config.maxBufferHole,
);
const waitingFragmentAtPosition = fragmentWithinToleranceTest(
bufferInfo.end,
this.config.maxFragLookUpTolerance,
frag,
);
if (waitingFragmentAtPosition < 0) {
this.log(
`Waiting fragment cc (${frag.cc}) @ ${frag.start} cancelled because another fragment at ${bufferInfo.end} is needed`,
);
this.clearWaitingFragment();
}
}
} else {
} else if (this.state !== State.STOPPED) {
this.state = State.IDLE;
}
}
Expand All @@ -259,9 +267,12 @@ class AudioStreamController
clearWaitingFragment() {
const waitingData = this.waitingData;
if (waitingData) {
if (!this.loadedmetadata) {
// Load overlapping fragment on start when discontinuity start times are not aligned
this.startFragRequested = false;
}
this.fragmentTracker.removeFragment(waitingData.frag);
this.waitingData = null;
this.waitingVideoCC = -1;
if (this.state !== State.STOPPED) {
this.state = State.IDLE;
}
Expand Down Expand Up @@ -343,12 +354,11 @@ class AudioStreamController

const fragments = trackDetails.fragments;
const start = fragments[0].start;
let targetBufferTime = this.flushing
? this.getLoadPosition()
: bufferInfo.end;
const loadPosition = this.getLoadPosition();
let targetBufferTime = this.flushing ? loadPosition : bufferInfo.end;

if (switchingTrack && media) {
const pos = this.getLoadPosition();
const pos = loadPosition;
// STABLE
if (
bufferedTrack &&
Expand Down Expand Up @@ -378,10 +388,8 @@ class AudioStreamController
}

let frag = this.getNextFragment(targetBufferTime, trackDetails);
let atGap = false;
// Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags
if (frag && this.isLoopLoading(frag, targetBufferTime)) {
atGap = !!frag.gap;
frag = this.getNextFragmentLoopLoading(
frag,
trackDetails,
Expand All @@ -395,39 +403,26 @@ class AudioStreamController
return;
}

if (!trackDetails.live || targetBufferTime < this.hls.liveSyncPosition!) {
if (
this.startFragRequested &&
(!trackDetails.live || targetBufferTime < this.hls.liveSyncPosition!)
) {
// Request audio segments up to one fragment ahead of main buffer
const mainBufferInfo = this.getFwdBufferInfo(
this.videoBuffer ? this.videoBuffer : this.media,
PlaylistLevelType.MAIN,
);
const mainFragLoading = this.mainFragLoading;
const mainTargetBufferEnd = mainFragLoading
? (mainFragLoading.part || mainFragLoading.frag).end
: null;
const atBufferSyncLimit =
!!mainBufferInfo && frag.start > mainBufferInfo.end + frag.duration;
if (atBufferSyncLimit) {
// Check fragment-tracker for main fragments since GAP segments do not show up in bufferInfo
const mainFrag = this.fragmentTracker.getFragAtPos(
frag.start,
PlaylistLevelType.MAIN,
);
if (mainFrag === null) {
return;
}
// Bridge gaps in main buffer (also prevents loop loading at gaps)
atGap ||= !!mainFrag.gap || mainBufferInfo.len === 0;
if (
!atGap ||
(bufferInfo.nextStart && bufferInfo.nextStart < mainFrag.end)
) {
return;
}
mainTargetBufferEnd !== null && frag.start > mainTargetBufferEnd;
if (atBufferSyncLimit && !frag.endList) {
return;
}
}

this.loadFragment(frag, levelInfo, targetBufferTime);
}

protected onMediaDetaching() {
this.videoBuffer = null;
this.bufferFlushed = this.flushing = false;
super.onMediaDetaching();
}
Expand Down Expand Up @@ -477,18 +472,16 @@ class AudioStreamController
}

protected onManifestLoading() {
this.fragmentTracker.removeAllFragments();
this.startPosition = this.lastCurrentTime = 0;
super.onManifestLoading();
this.bufferFlushed = this.flushing = false;
this.levels =
this.mainDetails =
this.mainDetails =
this.waitingData =
this.videoAnchor =
this.bufferedTrack =
this.cachedTrackLoadedData =
this.switchingTrack =
null;
this.startFragRequested = false;
this.trackId = this.videoTrackCC = this.waitingVideoCC = -1;
this.trackId = -1;
}

private onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
Expand Down Expand Up @@ -643,7 +636,6 @@ class AudioStreamController
complete: false,
});
cache.push(new Uint8Array(payload));
this.waitingVideoCC = this.videoTrackCC;
this.state = State.WAITING_INIT_PTS;
}
}
Expand All @@ -658,7 +650,7 @@ class AudioStreamController

private onBufferReset(/* event: Events.BUFFER_RESET */) {
// reset reference to sourcebuffers
this.mediaBuffer = this.videoBuffer = null;
this.mediaBuffer = null;
this.loadedmetadata = false;
}

Expand All @@ -670,21 +662,30 @@ class AudioStreamController
if (audioTrack) {
this.mediaBuffer = audioTrack.buffer || null;
}
if (data.tracks.video) {
this.videoBuffer = data.tracks.video.buffer || null;
}

private onFragLoading(event: Events.FRAG_LOADING, data: FragLoadingData) {
if (
data.frag.type === PlaylistLevelType.MAIN &&
data.frag.sn !== 'initSegment'
) {
this.mainFragLoading = data;
if (this.state === State.IDLE) {
this.tick();
}
}
}

private onFragBuffered(event: Events.FRAG_BUFFERED, data: FragBufferedData) {
const { frag, part } = data;
if (frag.type !== PlaylistLevelType.AUDIO) {
if (!this.loadedmetadata && frag.type === PlaylistLevelType.MAIN) {
const bufferable = this.videoBuffer || this.media;
if (bufferable) {
const bufferedTimeRanges = BufferHelper.getBuffered(bufferable);
if (bufferedTimeRanges.length) {
this.loadedmetadata = true;
}
const bufferedState = this.fragmentTracker.getState(frag);
if (
bufferedState === FragmentState.OK ||
bufferedState === FragmentState.PARTIAL
) {
this.loadedmetadata = true;
}
}
return;
Expand Down Expand Up @@ -889,12 +890,15 @@ class AudioStreamController
if (tracks.video) {
delete tracks.video;
}
if (tracks.audiovideo) {
delete tracks.audiovideo;
}

// include levelCodec in audio and video tracks
const track = tracks.audio;
if (!track) {
if (!tracks.audio) {
return;
}
const track = tracks.audio;

track.id = 'audio';

Expand All @@ -906,7 +910,7 @@ class AudioStreamController
if (variantAudioCodecs && variantAudioCodecs.split(',').length === 1) {
track.levelCodec = variantAudioCodecs;
}
this.hls.trigger(Events.BUFFER_CODECS, tracks);
this.hls.trigger(Events.BUFFER_CODECS, tracks as BufferCodecsData);
const initSegment = track.initSegment;
if (initSegment?.byteLength) {
const segment: BufferAppendingData = {
Expand All @@ -930,7 +934,6 @@ class AudioStreamController
) {
// only load if fragment is not loaded or if in audio switch
const fragState = this.fragmentTracker.getState(frag);
this.fragCurrent = frag;

// we force a frag loading in audio switch as fragment tracker might not have evicted previous frags in case of quick audio switch
if (
Expand All @@ -953,7 +956,6 @@ class AudioStreamController
alignMediaPlaylistByPDT(track.details, mainDetails);
}
} else {
this.startFragRequested = true;
super.loadFragment(frag, track, targetBufferTime);
}
} else {
Expand Down
Loading

0 comments on commit 20f50af

Please sign in to comment.