Home Reference Source

src/demux/transmuxer.ts

  1. import type { HlsEventEmitter } from '../events';
  2. import { Events } from '../events';
  3. import { ErrorTypes, ErrorDetails } from '../errors';
  4. import Decrypter from '../crypt/decrypter';
  5. import AACDemuxer from '../demux/aacdemuxer';
  6. import MP4Demuxer from '../demux/mp4demuxer';
  7. import TSDemuxer, { TypeSupported } from '../demux/tsdemuxer';
  8. import MP3Demuxer from '../demux/mp3demuxer';
  9. import MP4Remuxer from '../remux/mp4-remuxer';
  10. import PassThroughRemuxer from '../remux/passthrough-remuxer';
  11. import ChunkCache from './chunk-cache';
  12. import { appendUint8Array } from '../utils/mp4-tools';
  13. import { logger } from '../utils/logger';
  14. import type { Demuxer, DemuxerResult, KeyData } from '../types/demuxer';
  15. import type { Remuxer } from '../types/remuxer';
  16. import type { TransmuxerResult, ChunkMetadata } from '../types/transmuxer';
  17. import type { HlsConfig } from '../config';
  18. import type { LevelKey } from '../loader/level-key';
  19. import type { PlaylistLevelType } from '../types/loader';
  20.  
  21. let now;
  22. // performance.now() not available on WebWorker, at least on Safari Desktop
  23. try {
  24. now = self.performance.now.bind(self.performance);
  25. } catch (err) {
  26. logger.debug('Unable to use Performance API on this environment');
  27. now = self.Date.now;
  28. }
  29.  
  30. type MuxConfig =
  31. | { demux: typeof TSDemuxer; remux: typeof MP4Remuxer }
  32. | { demux: typeof MP4Demuxer; remux: typeof PassThroughRemuxer }
  33. | { demux: typeof AACDemuxer; remux: typeof MP4Remuxer }
  34. | { demux: typeof MP3Demuxer; remux: typeof MP4Remuxer };
  35.  
  36. const muxConfig: MuxConfig[] = [
  37. { demux: TSDemuxer, remux: MP4Remuxer },
  38. { demux: MP4Demuxer, remux: PassThroughRemuxer },
  39. { demux: AACDemuxer, remux: MP4Remuxer },
  40. { demux: MP3Demuxer, remux: MP4Remuxer },
  41. ];
  42.  
  43. let minProbeByteLength = 1024;
  44. muxConfig.forEach(({ demux }) => {
  45. minProbeByteLength = Math.max(minProbeByteLength, demux.minProbeByteLength);
  46. });
  47.  
  48. export default class Transmuxer {
  49. private observer: HlsEventEmitter;
  50. private typeSupported: TypeSupported;
  51. private config: HlsConfig;
  52. private vendor: string;
  53. private id: PlaylistLevelType;
  54. private demuxer?: Demuxer;
  55. private remuxer?: Remuxer;
  56. private decrypter?: Decrypter;
  57. private probe!: Function;
  58. private decryptionPromise: Promise<TransmuxerResult> | null = null;
  59. private transmuxConfig!: TransmuxConfig;
  60. private currentTransmuxState!: TransmuxState;
  61. private cache: ChunkCache = new ChunkCache();
  62.  
  63. constructor(
  64. observer: HlsEventEmitter,
  65. typeSupported: TypeSupported,
  66. config: HlsConfig,
  67. vendor: string,
  68. id: PlaylistLevelType
  69. ) {
  70. this.observer = observer;
  71. this.typeSupported = typeSupported;
  72. this.config = config;
  73. this.vendor = vendor;
  74. this.id = id;
  75. }
  76.  
  77. configure(transmuxConfig: TransmuxConfig) {
  78. this.transmuxConfig = transmuxConfig;
  79. if (this.decrypter) {
  80. this.decrypter.reset();
  81. }
  82. }
  83.  
  84. push(
  85. data: ArrayBuffer,
  86. decryptdata: LevelKey | null,
  87. chunkMeta: ChunkMetadata,
  88. state?: TransmuxState
  89. ): TransmuxerResult | Promise<TransmuxerResult> {
  90. const stats = chunkMeta.transmuxing;
  91. stats.executeStart = now();
  92.  
  93. let uintData: Uint8Array = new Uint8Array(data);
  94. const { cache, config, currentTransmuxState, transmuxConfig } = this;
  95. if (state) {
  96. this.currentTransmuxState = state;
  97. }
  98.  
  99. const keyData = getEncryptionType(uintData, decryptdata);
  100. if (keyData && keyData.method === 'AES-128') {
  101. const decrypter = this.getDecrypter();
  102. // Software decryption is synchronous; webCrypto is not
  103. if (config.enableSoftwareAES) {
  104. // Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached
  105. // data is handled in the flush() call
  106. const decryptedData = decrypter.softwareDecrypt(
  107. uintData,
  108. keyData.key.buffer,
  109. keyData.iv.buffer
  110. );
  111. if (!decryptedData) {
  112. stats.executeEnd = now();
  113. return emptyResult(chunkMeta);
  114. }
  115. uintData = new Uint8Array(decryptedData);
  116. } else {
  117. this.decryptionPromise = decrypter
  118. .webCryptoDecrypt(uintData, keyData.key.buffer, keyData.iv.buffer)
  119. .then((decryptedData): TransmuxerResult => {
  120. // Calling push here is important; if flush() is called while this is still resolving, this ensures that
  121. // the decrypted data has been transmuxed
  122. const result = this.push(
  123. decryptedData,
  124. null,
  125. chunkMeta
  126. ) as TransmuxerResult;
  127. this.decryptionPromise = null;
  128. return result;
  129. });
  130. return this.decryptionPromise!;
  131. }
  132. }
  133.  
  134. const {
  135. contiguous,
  136. discontinuity,
  137. trackSwitch,
  138. accurateTimeOffset,
  139. timeOffset,
  140. initSegmentChange,
  141. } = state || currentTransmuxState;
  142. const {
  143. audioCodec,
  144. videoCodec,
  145. defaultInitPts,
  146. duration,
  147. initSegmentData,
  148. } = transmuxConfig;
  149.  
  150. // Reset muxers before probing to ensure that their state is clean, even if flushing occurs before a successful probe
  151. if (discontinuity || trackSwitch || initSegmentChange) {
  152. this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration);
  153. }
  154.  
  155. if (discontinuity || initSegmentChange) {
  156. this.resetInitialTimestamp(defaultInitPts);
  157. }
  158.  
  159. if (!contiguous) {
  160. this.resetContiguity();
  161. }
  162.  
  163. if (this.needsProbing(uintData, discontinuity, trackSwitch)) {
  164. if (cache.dataLength) {
  165. const cachedData = cache.flush();
  166. uintData = appendUint8Array(cachedData, uintData);
  167. }
  168. this.configureTransmuxer(uintData, transmuxConfig);
  169. }
  170.  
  171. const result = this.transmux(
  172. uintData,
  173. keyData,
  174. timeOffset,
  175. accurateTimeOffset,
  176. chunkMeta
  177. );
  178. const currentState = this.currentTransmuxState;
  179.  
  180. currentState.contiguous = true;
  181. currentState.discontinuity = false;
  182. currentState.trackSwitch = false;
  183.  
  184. stats.executeEnd = now();
  185. return result;
  186. }
  187.  
  188. // Due to data caching, flush calls can produce more than one TransmuxerResult (hence the Array type)
  189. flush(
  190. chunkMeta: ChunkMetadata
  191. ): TransmuxerResult[] | Promise<TransmuxerResult[]> {
  192. const stats = chunkMeta.transmuxing;
  193. stats.executeStart = now();
  194.  
  195. const { decrypter, cache, currentTransmuxState, decryptionPromise } = this;
  196.  
  197. if (decryptionPromise) {
  198. // Upon resolution, the decryption promise calls push() and returns its TransmuxerResult up the stack. Therefore
  199. // only flushing is required for async decryption
  200. return decryptionPromise.then(() => {
  201. return this.flush(chunkMeta);
  202. });
  203. }
  204.  
  205. const transmuxResults: TransmuxerResult[] = [];
  206. const { timeOffset } = currentTransmuxState;
  207. if (decrypter) {
  208. // The decrypter may have data cached, which needs to be demuxed. In this case we'll have two TransmuxResults
  209. // This happens in the case that we receive only 1 push call for a segment (either for non-progressive downloads,
  210. // or for progressive downloads with small segments)
  211. const decryptedData = decrypter.flush();
  212. if (decryptedData) {
  213. // Push always returns a TransmuxerResult if decryptdata is null
  214. transmuxResults.push(
  215. this.push(decryptedData, null, chunkMeta) as TransmuxerResult
  216. );
  217. }
  218. }
  219.  
  220. const bytesSeen = cache.dataLength;
  221. cache.reset();
  222. const { demuxer, remuxer } = this;
  223. if (!demuxer || !remuxer) {
  224. // If probing failed, and each demuxer saw enough bytes to be able to probe, then Hls.js has been given content its not able to handle
  225. if (bytesSeen >= minProbeByteLength) {
  226. this.observer.emit(Events.ERROR, Events.ERROR, {
  227. type: ErrorTypes.MEDIA_ERROR,
  228. details: ErrorDetails.FRAG_PARSING_ERROR,
  229. fatal: true,
  230. reason: 'no demux matching with content found',
  231. });
  232. }
  233. stats.executeEnd = now();
  234. return [emptyResult(chunkMeta)];
  235. }
  236.  
  237. const demuxResultOrPromise = demuxer.flush(timeOffset);
  238. if (isPromise(demuxResultOrPromise)) {
  239. // Decrypt final SAMPLE-AES samples
  240. return demuxResultOrPromise.then((demuxResult) => {
  241. this.flushRemux(transmuxResults, demuxResult, chunkMeta);
  242. return transmuxResults;
  243. });
  244. }
  245.  
  246. this.flushRemux(transmuxResults, demuxResultOrPromise, chunkMeta);
  247. return transmuxResults;
  248. }
  249.  
  250. private flushRemux(
  251. transmuxResults: TransmuxerResult[],
  252. demuxResult: DemuxerResult,
  253. chunkMeta: ChunkMetadata
  254. ) {
  255. const { audioTrack, videoTrack, id3Track, textTrack } = demuxResult;
  256. const { accurateTimeOffset, timeOffset } = this.currentTransmuxState;
  257. logger.log(
  258. `[transmuxer.ts]: Flushed fragment ${chunkMeta.sn}${
  259. chunkMeta.part > -1 ? ' p: ' + chunkMeta.part : ''
  260. } of level ${chunkMeta.level}`
  261. );
  262. const remuxResult = this.remuxer!.remux(
  263. audioTrack,
  264. videoTrack,
  265. id3Track,
  266. textTrack,
  267. timeOffset,
  268. accurateTimeOffset,
  269. true,
  270. this.id
  271. );
  272. transmuxResults.push({
  273. remuxResult,
  274. chunkMeta,
  275. });
  276.  
  277. chunkMeta.transmuxing.executeEnd = now();
  278. }
  279.  
  280. resetInitialTimestamp(defaultInitPts: number | undefined) {
  281. const { demuxer, remuxer } = this;
  282. if (!demuxer || !remuxer) {
  283. return;
  284. }
  285. demuxer.resetTimeStamp(defaultInitPts);
  286. remuxer.resetTimeStamp(defaultInitPts);
  287. }
  288.  
  289. resetContiguity() {
  290. const { demuxer, remuxer } = this;
  291. if (!demuxer || !remuxer) {
  292. return;
  293. }
  294. demuxer.resetContiguity();
  295. remuxer.resetNextTimestamp();
  296. }
  297.  
  298. resetInitSegment(
  299. initSegmentData: Uint8Array | undefined,
  300. audioCodec: string | undefined,
  301. videoCodec: string | undefined,
  302. trackDuration: number
  303. ) {
  304. const { demuxer, remuxer } = this;
  305. if (!demuxer || !remuxer) {
  306. return;
  307. }
  308. demuxer.resetInitSegment(
  309. initSegmentData,
  310. audioCodec,
  311. videoCodec,
  312. trackDuration
  313. );
  314. remuxer.resetInitSegment(initSegmentData, audioCodec, videoCodec);
  315. }
  316.  
  317. destroy(): void {
  318. if (this.demuxer) {
  319. this.demuxer.destroy();
  320. this.demuxer = undefined;
  321. }
  322. if (this.remuxer) {
  323. this.remuxer.destroy();
  324. this.remuxer = undefined;
  325. }
  326. }
  327.  
  328. private transmux(
  329. data: Uint8Array,
  330. keyData: KeyData | null,
  331. timeOffset: number,
  332. accurateTimeOffset: boolean,
  333. chunkMeta: ChunkMetadata
  334. ): TransmuxerResult | Promise<TransmuxerResult> {
  335. let result: TransmuxerResult | Promise<TransmuxerResult>;
  336. if (keyData && keyData.method === 'SAMPLE-AES') {
  337. result = this.transmuxSampleAes(
  338. data,
  339. keyData,
  340. timeOffset,
  341. accurateTimeOffset,
  342. chunkMeta
  343. );
  344. } else {
  345. result = this.transmuxUnencrypted(
  346. data,
  347. timeOffset,
  348. accurateTimeOffset,
  349. chunkMeta
  350. );
  351. }
  352. return result;
  353. }
  354.  
  355. private transmuxUnencrypted(
  356. data: Uint8Array,
  357. timeOffset: number,
  358. accurateTimeOffset: boolean,
  359. chunkMeta: ChunkMetadata
  360. ): TransmuxerResult {
  361. const { audioTrack, videoTrack, id3Track, textTrack } = (
  362. this.demuxer as Demuxer
  363. ).demux(data, timeOffset, false, !this.config.progressive);
  364. const remuxResult = this.remuxer!.remux(
  365. audioTrack,
  366. videoTrack,
  367. id3Track,
  368. textTrack,
  369. timeOffset,
  370. accurateTimeOffset,
  371. false,
  372. this.id
  373. );
  374. return {
  375. remuxResult,
  376. chunkMeta,
  377. };
  378. }
  379.  
  380. private transmuxSampleAes(
  381. data: Uint8Array,
  382. decryptData: KeyData,
  383. timeOffset: number,
  384. accurateTimeOffset: boolean,
  385. chunkMeta: ChunkMetadata
  386. ): Promise<TransmuxerResult> {
  387. return (this.demuxer as Demuxer)
  388. .demuxSampleAes(data, decryptData, timeOffset)
  389. .then((demuxResult) => {
  390. const remuxResult = this.remuxer!.remux(
  391. demuxResult.audioTrack,
  392. demuxResult.videoTrack,
  393. demuxResult.id3Track,
  394. demuxResult.textTrack,
  395. timeOffset,
  396. accurateTimeOffset,
  397. false,
  398. this.id
  399. );
  400. return {
  401. remuxResult,
  402. chunkMeta,
  403. };
  404. });
  405. }
  406.  
  407. private configureTransmuxer(
  408. data: Uint8Array,
  409. transmuxConfig: TransmuxConfig
  410. ) {
  411. const { config, observer, typeSupported, vendor } = this;
  412. const {
  413. audioCodec,
  414. defaultInitPts,
  415. duration,
  416. initSegmentData,
  417. videoCodec,
  418. } = transmuxConfig;
  419. // probe for content type
  420. let mux;
  421. for (let i = 0, len = muxConfig.length; i < len; i++) {
  422. if (muxConfig[i].demux.probe(data)) {
  423. mux = muxConfig[i];
  424. break;
  425. }
  426. }
  427. if (!mux) {
  428. // If probing previous configs fail, use mp4 passthrough
  429. logger.warn(
  430. 'Failed to find demuxer by probing frag, treating as mp4 passthrough'
  431. );
  432. mux = { demux: MP4Demuxer, remux: PassThroughRemuxer };
  433. }
  434. // so let's check that current remuxer and demuxer are still valid
  435. const demuxer = this.demuxer;
  436. const remuxer = this.remuxer;
  437. const Remuxer: MuxConfig['remux'] = mux.remux;
  438. const Demuxer: MuxConfig['demux'] = mux.demux;
  439. if (!remuxer || !(remuxer instanceof Remuxer)) {
  440. this.remuxer = new Remuxer(observer, config, typeSupported, vendor);
  441. }
  442. if (!demuxer || !(demuxer instanceof Demuxer)) {
  443. this.demuxer = new Demuxer(observer, config, typeSupported);
  444. this.probe = Demuxer.probe;
  445. }
  446. // Ensure that muxers are always initialized with an initSegment
  447. this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration);
  448. this.resetInitialTimestamp(defaultInitPts);
  449. }
  450.  
  451. private needsProbing(
  452. data: Uint8Array,
  453. discontinuity: boolean,
  454. trackSwitch: boolean
  455. ): boolean {
  456. // in case of continuity change, or track switch
  457. // we might switch from content type (AAC container to TS container, or TS to fmp4 for example)
  458. return !this.demuxer || !this.remuxer || discontinuity || trackSwitch;
  459. }
  460.  
  461. private getDecrypter(): Decrypter {
  462. let decrypter = this.decrypter;
  463. if (!decrypter) {
  464. decrypter = this.decrypter = new Decrypter(this.observer, this.config);
  465. }
  466. return decrypter;
  467. }
  468. }
  469.  
  470. function getEncryptionType(
  471. data: Uint8Array,
  472. decryptData: LevelKey | null
  473. ): KeyData | null {
  474. let encryptionType: KeyData | null = null;
  475. if (
  476. data.byteLength > 0 &&
  477. decryptData != null &&
  478. decryptData.key != null &&
  479. decryptData.iv !== null &&
  480. decryptData.method != null
  481. ) {
  482. encryptionType = decryptData as KeyData;
  483. }
  484. return encryptionType;
  485. }
  486.  
  487. const emptyResult = (chunkMeta): TransmuxerResult => ({
  488. remuxResult: {},
  489. chunkMeta,
  490. });
  491.  
  492. export function isPromise<T>(p: Promise<T> | any): p is Promise<T> {
  493. return 'then' in p && p.then instanceof Function;
  494. }
  495.  
  496. export class TransmuxConfig {
  497. public audioCodec?: string;
  498. public videoCodec?: string;
  499. public initSegmentData?: Uint8Array;
  500. public duration: number;
  501. public defaultInitPts?: number;
  502.  
  503. constructor(
  504. audioCodec: string | undefined,
  505. videoCodec: string | undefined,
  506. initSegmentData: Uint8Array | undefined,
  507. duration: number,
  508. defaultInitPts?: number
  509. ) {
  510. this.audioCodec = audioCodec;
  511. this.videoCodec = videoCodec;
  512. this.initSegmentData = initSegmentData;
  513. this.duration = duration;
  514. this.defaultInitPts = defaultInitPts;
  515. }
  516. }
  517.  
  518. export class TransmuxState {
  519. public discontinuity: boolean;
  520. public contiguous: boolean;
  521. public accurateTimeOffset: boolean;
  522. public trackSwitch: boolean;
  523. public timeOffset: number;
  524. public initSegmentChange: boolean;
  525.  
  526. constructor(
  527. discontinuity: boolean,
  528. contiguous: boolean,
  529. accurateTimeOffset: boolean,
  530. trackSwitch: boolean,
  531. timeOffset: number,
  532. initSegmentChange: boolean
  533. ) {
  534. this.discontinuity = discontinuity;
  535. this.contiguous = contiguous;
  536. this.accurateTimeOffset = accurateTimeOffset;
  537. this.trackSwitch = trackSwitch;
  538. this.timeOffset = timeOffset;
  539. this.initSegmentChange = initSegmentChange;
  540. }
  541. }