index.js 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543
  1. // custom stringfly worker
  2. /* eslint-disable no-undef */
  3. const CACHE_BUF_LENGTH = 16;
  4. const YUV_BUF_LENGTH = 16;
  5. let testLastestTime = 0;
  6. if ("function" === typeof importScripts) {
  7. const startTime = Date.now();
  8. // self.importScripts('https://static.xverse.cn/wasm/zx_test_exclusive/v2/libxv265dec.js')
  9. // printConsole.log('Decoder update time is 2021/10/14 12:13 ')
  10. const YUVArray = [];
  11. const mediaArray = [];
  12. let IframesReceived = 0;
  13. let IframesDecoded = 0;
  14. let lastReceivePts = 0;
  15. let lastProcessPts = 0;
  16. let framesReturned = 0;
  17. let send_out_buffer = 0;
  18. let lastPoc = 0;
  19. let cachedFirstFrame = undefined;
  20. let cachedPanoramaFirstFrame = undefined;
  21. const printConsole = {
  22. log: (msg) =>
  23. self.postMessage({ t: MessageEvent.ConsoleLog, printMsg: msg }),
  24. error: (msg, code) =>
  25. self.postMessage({
  26. t: MessageEvent.ConsoleError,
  27. printMsg: msg,
  28. code: code,
  29. }),
  30. };
  31. const MessageEvent = {
  32. DecodeMessage: 0,
  33. UpdateStats: 1,
  34. WASMReady: 2,
  35. CacheFrame: 3,
  36. RecordVideo: 4,
  37. OnlyEmitSignal: 5,
  38. WASMReadyCost: 6,
  39. PanoramaMessage: 7,
  40. RequestIFrame: 8,
  41. ConsoleLog: 9,
  42. ConsoleError: 10,
  43. };
  44. let lastReceiveContentPts = 0;
  45. let saveMediaBytes = 0; // Just for test use
  46. const IFrameCacheBuffer = {};
  47. for (var i = 0; i < CACHE_BUF_LENGTH; ++i) {
  48. mediaArray.push({
  49. pts: -1,
  50. receive_ts: 0,
  51. decode_ts: 0,
  52. yuv_ts: 0,
  53. render_ts: 0,
  54. media: null,
  55. meta: null,
  56. isIDR: false,
  57. });
  58. }
  59. function Decoder() {
  60. this.expected_frameCnt = 1;
  61. this.inited = false;
  62. this.wasminited = false;
  63. this.cacheMap = new Map();
  64. this.receivedMedia = 0;
  65. this.receivedFrame = 0;
  66. this.receivedYUV = 0;
  67. this.receivedEmit = 0;
  68. this.lastReceivedEmit = 0;
  69. this.mediaBytesReceived = 0;
  70. this.metaBytesReceived = 0;
  71. this.prevSeq = 0;
  72. this.packetsLost = 0;
  73. this.packetsDrop = 0;
  74. this.dtpf = 0;
  75. this.dtmf = 0;
  76. this.getFrameInterval = 10;
  77. this.jumpI = false;
  78. this.startEmit = false;
  79. this.JankTimes = 0;
  80. this.bigJankTimes = 0;
  81. this.mediaCacheBuffer = new Uint8Array(1024 * 1024 * 10); // 10MB for video recording
  82. this.errorCacheBuffer = new Uint8Array(1024 * 1024 * 10); // 10MB for error stream recording
  83. this.mediaCacheSize = 0;
  84. this.errorCacheSize = 0;
  85. this.startRecord = false;
  86. this.saveRecord = false;
  87. this.requestingIFrame = false;
  88. this.decoderId = 0; // 0 for 720p, 1 for 480p.
  89. this.DecodablePts = 0;
  90. this.BlockedFrames = [];
  91. this.decodeTimeCircular = Array(120).fill(-1);
  92. this.dtcPtr = 0;
  93. this.readPtr = 1;
  94. this.writePtr = 1;
  95. this.cntBufInc = 0;
  96. this.prevBufNum = 0;
  97. this.MAX_TRY_TO_DEC_BUFNUM = 3;
  98. this.skipFrameUntilI = true;
  99. this.enable_logging = false;
  100. this.framesReceivedBetweenTimerInterval = 0;
  101. this.maxFramesReceivedBetweenTimerInterval = 0;
  102. this.isFirstFrame = 1;
  103. this.consumerPrevPts = -1;
  104. this.consumerCurrPts = -1;
  105. this.consumerWaitingIDR = false;
  106. this.lastObj = null;
  107. this.bufferIFrame = 0;
  108. this.passiveJitterLength = 0;
  109. }
  110. //refactor:
  111. Decoder.prototype.isBufEmpty = function () {
  112. return this.readPtr == this.writePtr;
  113. };
  114. Decoder.prototype.isBufFull = function () {
  115. return (this.writePtr + 1) % CACHE_BUF_LENGTH == this.readPtr;
  116. };
  117. Decoder.prototype.getNumOfPktToBeDec = function () {
  118. return (this.writePtr + CACHE_BUF_LENGTH - this.readPtr) % CACHE_BUF_LENGTH;
  119. };
  120. Decoder.prototype.getNumOfEmptySlot = function () {
  121. return CACHE_BUF_LENGTH - this.getNumOfPktToBeDec() - 1;
  122. };
  123. Decoder.prototype.aheadof = function (a, b) {
  124. return (a - b + 65536) % 65536 > 65536 / 2;
  125. };
  126. Decoder.prototype.distance = function (a, b) {
  127. var res;
  128. if (this.aheadof(a, b)) {
  129. res = this.seqDiff(b, a, 65536);
  130. } else {
  131. res = this.seqDiff(a, b, 65536);
  132. }
  133. return res;
  134. };
  135. Decoder.prototype.isSeqJump = function (a, b) {
  136. return this.distance(a, b) >= CACHE_BUF_LENGTH - 1;
  137. };
  138. Decoder.prototype.seqDiff = function (a, b, mod) {
  139. return (a + mod - b) % mod;
  140. };
  141. //notice: n could be nagative
  142. Decoder.prototype.seqAdd = function (seq, n, mod) {
  143. return (seq + mod + n) % mod;
  144. };
  145. //end refactor
  146. Decoder.prototype.resetDecoder = function () {
  147. this.isFirstFrame = 1;
  148. this.expected_frameCnt = 1;
  149. this.receivedMedia = 0;
  150. this.receivedYUV = 0;
  151. this.receivedEmit = 0;
  152. this.lastReceivedEmit = 0;
  153. this.mediaBytesReceived = 0;
  154. this.metaBytesReceived = 0;
  155. this.prevSeq = 0;
  156. this.packetsLost = 0;
  157. this.packetsDrop = 0;
  158. this.dtpf = 0;
  159. this.dtmf = 0;
  160. this.JankTimes = 0;
  161. this.bigJankTimes = 0;
  162. this.getFrameInterval = 10;
  163. this.jumpI = false;
  164. IframesReceived = 0;
  165. IframesDecoded = 0;
  166. lastReceivePts = 0;
  167. lastProcessPts = 0;
  168. lastReceiveContentPts = 0;
  169. this.requestingIFrame = false;
  170. this.DecodablePts = 0;
  171. this.BlockedFrames = [];
  172. this.decodeTimeCircular.fill(-1);
  173. this.dtcPtr = 0;
  174. for (var i = 0; i < CACHE_BUF_LENGTH; ++i) {
  175. mediaArray[i].media = null;
  176. mediaArray[i].meta = null;
  177. mediaArray[i] = {
  178. pts: -1,
  179. receive_ts: 0,
  180. decode_ts: 0,
  181. yuv_ts: 0,
  182. render_ts: 0,
  183. media: null,
  184. meta: null,
  185. isIDR: false,
  186. };
  187. }
  188. //refactor:
  189. this.readPtr = this.writePtr = 1;
  190. this.cntBufInc = 0;
  191. this.prevBufNum = 0;
  192. this.MAX_TRY_TO_DEC_BUFNUM = 3;
  193. this.skipFrameUntilI = true;
  194. this.consumerPrevPts = -1;
  195. this.consumerCurrPts = -1;
  196. this.consumerWaitingIDR = false;
  197. this.lastObj = null;
  198. this.bufferIFrame = 0;
  199. //end refactor
  200. };
  201. //refactor:
  202. Decoder.prototype.changeLogSwitch = function (status) {
  203. this.enable_logging = status;
  204. };
  205. const MAX_LOG_NUM = 128;
  206. logBufQueue = [];
  207. Decoder.prototype.dumpLogBuf = function () {
  208. while (logBufQueue.length > 0) {
  209. console.log(logBufQueue.shift());
  210. }
  211. };
  212. Decoder.prototype.dumpJitterBufInfo = function (label, pts = -1) {
  213. // if (!this.enable_logging) {
  214. // return
  215. // }
  216. logInfo =
  217. "WritePtr: " +
  218. this.writePtr +
  219. ", ReadPtr: " +
  220. this.readPtr +
  221. "\\n" +
  222. ", Producer Prev/Curr: " +
  223. this.prevSeq +
  224. "/" +
  225. pts +
  226. "\\n" +
  227. ", Consumer Prev/Curr: " +
  228. this.consumerPrevPts +
  229. "/" +
  230. this.consumerCurrPts +
  231. "\\n" +
  232. "awaitingBuf: " +
  233. this.getNumOfPktToBeDec() +
  234. ", emptySlotNum: " +
  235. this.getNumOfEmptySlot() +
  236. ", skipFrameUntilI: " +
  237. this.skipFrameUntilI +
  238. "\\n" +
  239. " framesReceivedBetweenTimerInterval: " +
  240. this.framesReceivedBetweenTimerInterval +
  241. ", maxFramesReceivedBetweenTimerInterval: " +
  242. this.maxFramesReceivedBetweenTimerInterval +
  243. "\\n" +
  244. " label: " +
  245. label +
  246. "\\n";
  247. if (pts != -1) {
  248. logInfo +=
  249. " this.notEnoughSlots(" +
  250. pts +
  251. "): " +
  252. this.notEnoughSlots(pts) +
  253. "\\n";
  254. }
  255. if (this.enable_logging) {
  256. console.log(logInfo);
  257. } else {
  258. logBufQueue.push(logInfo);
  259. if (logBufQueue.length > MAX_LOG_NUM) {
  260. logBufQueue.shift();
  261. }
  262. }
  263. };
  264. Decoder.prototype.resetBufItem = function (index) {
  265. mediaArray[index].media = null;
  266. mediaArray[index].meta = null;
  267. if (mediaArray[index].isIDR == true) {
  268. this.bufferIFrame -= 1;
  269. }
  270. mediaArray[index] = {
  271. pts: -1,
  272. receive_ts: 0,
  273. decode_ts: 0,
  274. yuv_ts: 0,
  275. render_ts: 0,
  276. media: null,
  277. meta: null,
  278. isIDR: false,
  279. };
  280. this.readPtr = this.seqAdd(this.readPtr, 1, CACHE_BUF_LENGTH);
  281. };
  282. Decoder.prototype.checkPktOrderInConsumer = function (index) {
  283. if (this.consumerPrevPts == -1) {
  284. if (!this.isSlotEmpty(index)) {
  285. this.consumerPrevPts = mediaArray[index].pts;
  286. }
  287. return true;
  288. }
  289. if (this.isSlotEmpty(index)) {
  290. //lost
  291. // debugger
  292. console.log("[xmedia] return on SLOT EMPTY, prev: %s", prevSeq);
  293. this.consumerWaitingIDR = true;
  294. this.consumerPrevPts = this.seqAdd(this.consumerPrevPts, 1, 65536);
  295. return true;
  296. }
  297. if (!this.slotHasMedia(index)) {
  298. // pure meta
  299. // debugger
  300. console.log(
  301. "[xmedia] return on meta, prev: %s, cur: %s",
  302. mediaArray[index].pts
  303. );
  304. this.consumerPrevPts = mediaArray[index].pts;
  305. return true;
  306. }
  307. this.consumerCurrPts = mediaArray[index].pts;
  308. if (
  309. this.consumerWaitingIDR ||
  310. this.seqDiff(this.consumerCurrPts, this.consumerPrevPts, 65536) != 1
  311. ) {
  312. // if (!mediaArray[index].isIDR && mediaArray[index].media.byteLength!=0) {
  313. if (
  314. this.isPFrame(
  315. mediaArray[index].isIDR,
  316. mediaArray[index].media.byteLength
  317. )
  318. ) {
  319. console.error(
  320. "[INFO][XMEDIA] optimize to further reduce clutter chance. copy console log to developer"
  321. );
  322. this.dumpLogBuf();
  323. this.dumpJitterBufInfo("go away.");
  324. // debugger
  325. this.consumerPrevPts = -1;
  326. // this.resetDecoder()
  327. return false;
  328. }
  329. }
  330. console.log(
  331. "[xmedia] return finally, prev: %s, cur: %s",
  332. this.prevSeq,
  333. this.consumerCurrPts,
  334. Date.now() - testLastestTime
  335. );
  336. testLastestTime = String(Date.now()).slice();
  337. this.consumerPrevPts = this.consumerCurrPts;
  338. this.consumerWaitingIDR = false;
  339. return true;
  340. };
  341. Decoder.prototype.slotHasMedia = function (index) {
  342. return (
  343. mediaArray[index].media != null && mediaArray[index].media.byteLength != 0
  344. );
  345. };
  346. Decoder.prototype.slotHasContent = function (index) {
  347. return (
  348. mediaArray[index].media != null &&
  349. mediaArray[index].meta != null &&
  350. mediaArray[index].pts != -1
  351. );
  352. };
  353. Decoder.prototype.procBufItem = function (index) {
  354. this.dumpJitterBufInfo("Entering Decoder.prototype.procBufItem");
  355. // console.log("[][Core][WASM], pts: %s, isIDR: %s, length: %s",
  356. // mediaArray[index].pts,
  357. // mediaArray[index].isIDR,
  358. // mediaArray[index].media.length
  359. // );
  360. console.log("metaverse----core, pts: %s, isIDR: %s, length: %s",
  361. mediaArray[index].pts,
  362. mediaArray[index].isIDR,
  363. mediaArray[index].media.length
  364. );
  365. // var loginfo = 'pts: %s, isIDR: %s, length: %s', mediaArray[index].pts, mediaArray[index].isIDR, mediaArray[index].media.length
  366. needToSkip = this.skipFrameUntilI && !mediaArray[index].isIDR;
  367. var loginfo =
  368. "index:" +
  369. index +
  370. ", pts: " +
  371. mediaArray[index].pts +
  372. ", isidr: " +
  373. mediaArray[index].isIDR +
  374. ", slotHasMedia: " +
  375. this.slotHasMedia(index) +
  376. ", slotHasMeta: " +
  377. (mediaArray[index].meta != null) +
  378. ", needToSkip: " +
  379. needToSkip;
  380. // console.log("procBufItem", index, mediaArray[index]);
  381. if (this.slotHasContent(index) && !needToSkip) {
  382. console.log("[xmedia] %s ------------ 001", mediaArray[index].pts);
  383. let objData = {
  384. media: mediaArray[index].media,
  385. frameCnt: mediaArray[index].pts,
  386. meta: mediaArray[index].meta,
  387. metadata: mediaArray[index].metadata,
  388. isIDR: mediaArray[index].isIDR,
  389. };
  390. // -------------------
  391. if (this.checkPktOrderInConsumer(index)) {
  392. console.log("[xmedia] %s ------------ 002", mediaArray[index].pts);
  393. this.decodeFrame(objData);
  394. }
  395. if (mediaArray[index].isIDR) {
  396. console.log("[xmedia] %s ------------ 003", mediaArray[index].pts);
  397. // console.log("mediaArray[index].isIDR: this.skipFrameUntilI = false")
  398. this.skipFrameUntilI = false;
  399. }
  400. } else {
  401. console.log("[xmedia] %s ------------ 004", mediaArray[index].pts);
  402. if (this.slotHasMedia(index)) {
  403. console.log("[xmedia] %s ------------ 005", mediaArray[index].pts);
  404. //need to skip, waiting I Frame
  405. //dropCache++
  406. this.dropPkt += 1;
  407. // MARKER META1META2
  408. // self.postMessage({ t: MessageEvent.OnlyEmitSignal, meta_only: true, meta: mediaArray[index].meta, metadata: mediaArray[index].metadata })
  409. } else {
  410. console.log("[xmedia] %s ------------ 006", mediaArray[index].pts);
  411. // no media
  412. if (mediaArray[index].meta != null) {
  413. this.checkPktOrderInConsumer(index);
  414. console.log("[xmedia] %s ------------ 007", mediaArray[index].pts);
  415. // Still frame
  416. console.log("[send signal]", mediaArray[index].pts);
  417. self.postMessage({
  418. t: MessageEvent.OnlyEmitSignal,
  419. meta_only: true,
  420. meta: mediaArray[index].meta,
  421. metadata: mediaArray[index].metadata,
  422. });
  423. } else {
  424. console.log("[xmedia] %s ------------ 008", mediaArray[index].pts);
  425. // Lost_rcv++
  426. // console.log("lost_rcv++: this.skipFrameUntilI = true")
  427. // console.info('[xmedia] FFFFF This code should not be executed!!!!')
  428. console.info(
  429. "[xmedia] null pkt sneaked into profBufItem without harm"
  430. );
  431. this.skipFrameUntilI = true;
  432. }
  433. }
  434. }
  435. this.dumpJitterBufInfo("Leaving Decoder.prototype.procBufItem, " + loginfo);
  436. this.lastObj = mediaArray[index];
  437. this.resetBufItem(index);
  438. };
  439. Decoder.prototype.flushBuffer = function (untilIDR) {
  440. this.dumpJitterBufInfo("Entering Decoder.prototype.flushBuffer");
  441. this.skipFrameUntilI = true;
  442. var breakWhenIDR = false;
  443. while (this.getNumOfPktToBeDec() > 0) {
  444. index = this.readPtr;
  445. if (this.slotHasMedia(index)) {
  446. // dropMedia until IDR // \u6765\u4E0D\u53CA\u89E3\u7801\u4E22\u5E27
  447. this.packetsDrop += 1;
  448. if (untilIDR) {
  449. if (mediaArray[index].isIDR == true) {
  450. breakWhenIDR = true;
  451. break;
  452. }
  453. }
  454. } else if (mediaArray[index].meta != null) {
  455. self.postMessage({
  456. t: MessageEvent.OnlyEmitSignal,
  457. meta_only: true,
  458. meta: mediaArray[index].meta,
  459. metadata: mediaArray[index].metadata,
  460. });
  461. }
  462. this.resetBufItem(index);
  463. }
  464. if (!breakWhenIDR) {
  465. this.isFirstFrame = true;
  466. }
  467. this.dumpJitterBufInfo("Leaving Decoder.prototype.flushBuffer");
  468. return this.isFirstFrame;
  469. };
  470. // var cnt = 0
  471. Decoder.prototype.getFrameToDecode = function () {
  472. this.dumpJitterBufInfo("Entering Decoder.prototype.getFrameToDecode");
  473. if (this.getNumOfPktToBeDec() == 0) {
  474. return false;
  475. }
  476. //bufNum awaiting increase counter
  477. // while (this.getNumOfPktToBeDec() > CACHE_BUF_LENGTH / 2) {
  478. // needToCheck = true
  479. // if (this.cntBufInc > this.MAX_TRY_TO_DEC_BUFNUM) {
  480. // console.log('ringbuffer is deteriorating, flush until IDR')
  481. // var untilIDR = true
  482. // this.flushBuffer(untilIDR)
  483. // this.cntBufInc = 0
  484. // break
  485. // }
  486. // this.procBufItem(this.readPtr)
  487. // }
  488. // if (this.getNumOfPktToBeDec() == 0) {
  489. // return false
  490. // }
  491. let IFrmInBuffer = 0;
  492. let frmInBuffer = 0;
  493. for (var i = 0; i < CACHE_BUF_LENGTH; ++i) {
  494. if (mediaArray[i].isIDR) {
  495. IFrmInBuffer += 1;
  496. }
  497. if (this.slotHasMedia(i)) {
  498. frmInBuffer += 1;
  499. }
  500. }
  501. if (!this.slotHasContent(this.readPtr) && IFrmInBuffer == 0) {
  502. if (frmInBuffer > 0) {
  503. // There is P frame in buffer but cannot be decoded.
  504. // Due to ordered data channel, this is packet loss.
  505. // So request for I frame here.
  506. printConsole.log("detect packet lost. Request for I frame.");
  507. self.postMessage({ t: MessageEvent.RequestIFrame });
  508. }
  509. return false;
  510. }
  511. this.procBufItem(this.readPtr);
  512. // if (this.getNumOfPktToBeDec() > this.prevBufNum) {
  513. // this.cntBufInc++
  514. // } else {
  515. // if (this.cntBufInc > 2) {
  516. // // aimd
  517. // this.cntBufInc / 2
  518. // }
  519. // }
  520. // this.prevBufNum = this.getNumOfPktToBeDec()
  521. this.dumpJitterBufInfo("Leaving Decoder.prototype.getFrameToDecode");
  522. return true;
  523. };
  524. //refactor end:
  525. var cacheBuffer;
  526. var resultBuffer;
  527. Decoder.prototype.startDecoding = function () {
  528. function iterative_getFrameToDecode() {
  529. self.decoder.framesReceivedBetweenTimerInterval = 0;
  530. self.decoder.dumpJitterBufInfo(
  531. "Entering Decoder.prototype.iterative_getFrameToDecode"
  532. );
  533. var start_ts = Date.now();
  534. let hasDecodeFrame = self.decoder.getFrameToDecode();
  535. var end_ts = Date.now();
  536. // refactor
  537. let expect_interval =
  538. 1000 /
  539. (30 +
  540. Math.max(
  541. self.decoder.getNumOfPktToBeDec() -
  542. self.decoder.passiveJitterLength,
  543. 0
  544. ));
  545. //let expect_interval = 1000 / (Decoder.prototype.getNumOfPktToBeDec() + 30)
  546. if (hasDecodeFrame) {
  547. let usedTime = end_ts - start_ts;
  548. self.decoder.getFrameInterval =
  549. expect_interval - Math.max(usedTime, self.decoder.dtpf);
  550. if (self.decoder.getFrameInterval < 1) {
  551. self.decoder.getFrameInterval = 0;
  552. }
  553. } else {
  554. self.decoder.getFrameInterval = 5;
  555. }
  556. // let usedTime = end_ts - start_ts
  557. // FPS = 30
  558. // if (usedTime * FPS < 1000) {
  559. // self.decoder.getFrameInterval = 1000 / (FPS + Decoder.prototype.getNumOfPktToBeDec())
  560. // } else {
  561. // self.decoder.getFrameInterval = 1 //ms
  562. // }
  563. // if (Decoder.prototype.getNumOfPktToBeDec() == 0) {
  564. // //Hinse: have to get buf to send asap.
  565. // self.decoder.getFrameInterval = 5 //ms
  566. // }
  567. setTimeout(iterative_getFrameToDecode, self.decoder.getFrameInterval);
  568. self.decoder.dumpJitterBufInfo(
  569. "Leaving Decoder.prototype.iterative_getFrameToDecode"
  570. );
  571. // refactor end
  572. }
  573. function postStats() {
  574. function add(accumulator, a) {
  575. if (a == -1) {
  576. a = 0;
  577. }
  578. return accumulator + a;
  579. }
  580. function count_valid(accumulator, a) {
  581. let non_zero = 0;
  582. if (a != -1) {
  583. non_zero = 1;
  584. }
  585. return accumulator + non_zero;
  586. }
  587. function max(maxer, a) {
  588. return Math.max(maxer, a);
  589. }
  590. const dtpf =
  591. self.decoder.decodeTimeCircular.reduce(add, 0) /
  592. self.decoder.decodeTimeCircular.reduce(count_valid, 0) || 0;
  593. const dtmf = self.decoder.decodeTimeCircular.reduce(max, 0);
  594. let objData = {
  595. t: MessageEvent.UpdateStats,
  596. mediaBytesReceived: self.decoder.mediaBytesReceived,
  597. metaBytesReceived: self.decoder.metaBytesReceived,
  598. packetsLost: self.decoder.packetsLost, // \u7F51\u7EDC\u4E22\u5E27
  599. packetsDrop: self.decoder.packetsDrop, // \u6765\u4E0D\u53CA\u89E3\u7801\u4E22\u5E27
  600. framesReceived: self.decoder.receivedMedia,
  601. framesDecoded: self.decoder.receivedYUV,
  602. framesRendered: self.decoder.receivedEmit,
  603. framesReturned: framesReturned,
  604. // framesAwait: leastReceivePts - lastProcessPts,
  605. framesAwait: self.decoder.getNumOfPktToBeDec(), // \u7B49\u5F85\u89E3\u7801\u7684\u5E27
  606. decodeTimePerFrame: dtpf,
  607. decodeTimeMaxFrame: dtmf,
  608. sendOutBuffer: send_out_buffer,
  609. JankTimes: self.decoder.JankTimes,
  610. bigJankTimes: self.decoder.bigJankTimes,
  611. receivedIframe: self.decoder.IframesReceived,
  612. decodedIframe: self.decoder.IframesDecoded,
  613. };
  614. self.postMessage(objData);
  615. self.decoder.dtmf = 0;
  616. }
  617. setTimeout(iterative_getFrameToDecode, this.getFrameInterval);
  618. setInterval(postStats, 1000);
  619. };
  620. Decoder.prototype.initAll = function (config) {
  621. if (typeof wasmSource != "undefined") {
  622. if (wasmSource == 0) {
  623. // Load from indexedDB
  624. // console.log('Load WASM from indexedDB')
  625. printConsole.log("Load WASM from indexedDB");
  626. wasmSource = undefined;
  627. } else if (wasmSource == 1) {
  628. // Load by fetch
  629. // console.log('Load WASM by fetch')
  630. printConsole.log("Load WASM by fetch");
  631. wasmSource = undefined;
  632. } else {
  633. printConsole.log("WASM not ready now, wait for 200 ms.");
  634. }
  635. } else {
  636. printConsole.log(
  637. "wasm variable is not defined. Probably libffmpeg.js file is not loaded properly."
  638. );
  639. }
  640. if (typeof wasmTable === "undefined") {
  641. setTimeout(self.decoder.initAll, 200, config);
  642. return 0;
  643. }
  644. cacheBuffer = Module._malloc(1024 * 1024);
  645. resultBuffer = Module._malloc(64);
  646. self.postMessage({
  647. t: MessageEvent.WASMReadyCost,
  648. type: "report",
  649. data: {
  650. metric: "wasmDownloadCost",
  651. value: Date.now() - startTime,
  652. group: "costs",
  653. },
  654. });
  655. // WASM already initialized. Now we open decoder.
  656. const LOG_LEVEL_WASM = 2;
  657. const DECODER_H264 = 0;
  658. const decoder_type = DECODER_H264;
  659. for (var j = 0; j < YUV_BUF_LENGTH; ++j) {
  660. YUVArray.push({
  661. status: 0,
  662. buffer: new Uint8Array((config.width * config.height * 3) / 2),
  663. });
  664. }
  665. printConsole.log("Going to open decoder " + String(Date.now()));
  666. var ret0 = Module._openDecoder(0, decoder_type, LOG_LEVEL_WASM);
  667. if (ret0 == 0) {
  668. self.decoder.startDecoding(); //开始解码
  669. self.postMessage({
  670. t: MessageEvent.WASMReady,
  671. wasm_ready: true,
  672. updateStats: false,
  673. });
  674. } else {
  675. printConsole.error(
  676. "openDecoder failed with error " + String(ret0),
  677. "5001"
  678. );
  679. return 1;
  680. }
  681. return 0;
  682. };
  683. Decoder.prototype.cacheFrame = function (data) {
  684. if (data.position != undefined) {
  685. var media = data.data.subarray(
  686. data.metaLen,
  687. data.metaLen + data.mediaLen
  688. );
  689. if (IFrameCacheBuffer[JSON.stringify(data.position)] == undefined) {
  690. for (var key in IFrameCacheBuffer) delete IFrameCacheBuffer[key]; // Clear Frame Cache
  691. IFrameCacheBuffer[JSON.stringify(data.position)] = {};
  692. }
  693. IFrameCacheBuffer[JSON.stringify(data.position)][data.cachedKey] = media;
  694. self.postMessage({
  695. t: MessageEvent.CacheFrame,
  696. cacheFrame: true,
  697. cachedKey: data.cachedKey,
  698. metadata: data.metadata,
  699. });
  700. }
  701. };
  702. Decoder.prototype.updateMediaMetaStats = function (data) {
  703. this.metaBytesReceived += data.metaLen;
  704. this.mediaBytesReceived += data.mediaLen;
  705. if (data.mediaLen != 0) {
  706. this.receivedMedia++;
  707. }
  708. };
  709. Decoder.prototype.isIFrame = function (isIDR, mediaLen) {
  710. // return data.isIDR && media.byteLength !=0
  711. let flag = isIDR && mediaLen != 0;
  712. if(flag){
  713. console.log('metaverse---------是IFrame');
  714. }
  715. else{
  716. console.log('metaverse---------不是IFrame');
  717. }
  718. return flag;
  719. };
  720. Decoder.prototype.isPFrame = function (isIDR, mediaLen) {
  721. // return !data.isIDR && media.byteLength !=0
  722. let flag = !isIDR && mediaLen != 0;
  723. if(flag){
  724. console.log('metaverse---------是PFrame');
  725. }
  726. else{
  727. console.log('metaverse---------不是PFrame');
  728. }
  729. return flag;
  730. };
  731. Decoder.prototype.isPureMeta = function (metaLen, mediaLen) {
  732. // return media.byteLength == 0 && meta.byteLength !=0
  733. let flag = mediaLen == 0 && metaLen != 0;
  734. if(flag){
  735. console.log('metaverse---------是PureMeta');
  736. }
  737. else{
  738. console.log('metaverse---------不是PureMeta');
  739. }
  740. return flag;
  741. };
  742. Decoder.prototype.isInvalidPkt = function (isIDR, mediaLen, metaLen) {
  743. return (
  744. !this.isIFrame(isIDR, mediaLen) &&
  745. !this.isPFrame(isIDR, mediaLen) &&
  746. !this.isPureMeta(metaLen, mediaLen)
  747. );
  748. };
  749. Decoder.prototype.isSlotEmpty = function (index) {
  750. return !this.slotHasMedia(index) && mediaArray[index].meta == null;
  751. };
  752. Decoder.prototype.handleNewPktOnFlush = function (isIDR, mediaLen) {
  753. var dropPkt = false;
  754. // console.log("[xmedia] 000-1 isFirstFrame %s", this.isFirstFrame)
  755. if (this.isFirstFrame) {
  756. // let IDR/meta pass
  757. // console.log("[xmedia] 000-2 isIDR: %s, mediaLen: %s", isIDR, mediaLen)
  758. // console.log("[xmedia] 000-3 this.isPFrame(isIDR, mediaLen): %s", this.isPFrame(isIDR, mediaLen))
  759. if (this.isPFrame(isIDR, mediaLen)) {
  760. console.log("[xmedia] 001: isPFrame TRUE");
  761. this.packetsDrop += 1;
  762. // MARKER META1META2
  763. dropPkt = true;
  764. }
  765. if (this.isIFrame(isIDR, mediaLen)) {
  766. console.log("[xmedia] 002: isIFrame TRUE");
  767. this.isFirstFrame = false;
  768. }
  769. }
  770. console.log("[xmedia] 003: dropPkt: %s", dropPkt);
  771. return dropPkt;
  772. };
  773. Decoder.prototype.notEnoughSlots = function (pts) {
  774. return (
  775. this.isBufFull() ||
  776. this.seqDiff(pts, this.prevSeq, CACHE_BUF_LENGTH) >
  777. this.getNumOfEmptySlot()
  778. );
  779. };
  780. Decoder.prototype.receiveFrame = function (data) {
  781. var key = data.cachedKey;
  782. var pts = data.frameCnt;
  783. var meta = data.data.subarray(0, data.metaLen);
  784. var media;
  785. if (data.cached) {
  786. media = IFrameCacheBuffer[JSON.stringify(data.position)][key];
  787. } else if (data.cacheRequest) {
  788. media = data.data.subarray(data.metaLen, data.metaLen + data.mediaLen);
  789. self.decoder.cacheFrame(data);
  790. } else {
  791. media = data.data.subarray(data.metaLen, data.metaLen + data.mediaLen);
  792. }
  793. // console.log("[xmedia] media1", media);
  794. // this.updateMediaMetaStats(data);
  795. if (this.isFirstFrame) {
  796. console.log("[xmedia] isFirstFrame = true. pts:%s", pts);
  797. if (this.isPFrame(data.isIDR, media.byteLength)) {
  798. // MARKER META1META2
  799. this.packetsDrop += 1;
  800. console.log("fk1");
  801. return;
  802. }
  803. this.prevSeq = this.seqDiff(pts, 1, 65536);
  804. this.readPtr = this.writePtr = pts % CACHE_BUF_LENGTH;
  805. if (data.isIDR) {
  806. this.isFirstFrame = false;
  807. }
  808. }
  809. if (pts !== this.seqAdd(this.prevSeq, 1, 65536) && pts !== this.prevSeq) {
  810. this.packetsLost += 1;
  811. }
  812. const index = pts % CACHE_BUF_LENGTH;
  813. if (this.startRecord) {
  814. this.mediaCacheBuffer.set(media, this.mediaCacheSize);
  815. this.mediaCacheSize += media.byteLength;
  816. }
  817. if (this.saveRecord) {
  818. downloadBlob(
  819. this.mediaCacheBuffer.subarray(0, this.mediaCacheSize),
  820. "test.264",
  821. "application/octet-stream"
  822. );
  823. this.mediaCacheSize = 0;
  824. this.saveRecord = false;
  825. this.startRecord = false;
  826. }
  827. //refactor:
  828. // Step 1, big jump detected. we cannot handle it, flush all.
  829. var untilIDR, pktDrop;
  830. if (this.isSeqJump(this.prevSeq, pts)) {
  831. // console.log('[resetdecoder] Fatal: decoder seq jump from ' + this.prevSeq + ' to ' + pts)
  832. untilIDR = false;
  833. this.flushBuffer(untilIDR);
  834. pktDrop = this.handleNewPktOnFlush(data.isIDR, media.byteLength);
  835. if (pktDrop) return;
  836. }
  837. this.dumpJitterBufInfo("Entering Decoder.prototype.receiveFrame", pts);
  838. // console.log("--->> this.notEnoughSlots(pts): %s", this.notEnoughSlots(pts))
  839. // Step 2,
  840. if (this.aheadof(pts, this.prevSeq)) {
  841. // pts before prevSeq
  842. // pkts in wrong order
  843. if (this.packetsLost > 0) {
  844. this.packetsLost -= 1;
  845. // this.packetdisorder +=1
  846. }
  847. console.log(
  848. "[xmedia] disorder frame received. preSeq: %s, pts: %s",
  849. this.prevSeq,
  850. pts
  851. );
  852. if (this.seqDiff(this.prevSeq, pts, 65536) < this.getNumOfPktToBeDec()) {
  853. // slot for pts is not handled yet. just put it back:
  854. console.log(
  855. "put disorder frame to enc_queue, pkt:%s, prevPts: %s, numOfPktToBeDec: %s",
  856. pts,
  857. this.prevSeq,
  858. this.getNumOfPktToBeDec()
  859. );
  860. } else {
  861. //dropDisorder++
  862. console.error(
  863. "drop disorder pkt:%s, prevPts: %s, numOfPktToBeDec: %s",
  864. pts,
  865. this.prevSeq,
  866. this.getNumOfPktToBeDec()
  867. );
  868. this.packetsDrop += 1;
  869. // ---------------------
  870. // Note:
  871. //
  872. // Three principles for meta data:
  873. // step 1 step 2
  874. // 1. backend -----> frontend (decoder.js) -----> frontend (worker.js), meta pkts must be kept in order in the whole pipeline
  875. // 2. if media presents and needs to be dropped, the meta companion needs to be dropped together.
  876. // 3. if media is absent (media.bytelength == 0), send meta anyway
  877. // ---------------------
  878. // According to rule 1, drop meta at this point is reasonable.
  879. return;
  880. }
  881. } else {
  882. // pts after prevSeq
  883. // make sure the ringbuffer has empty slot for new pkt
  884. if (this.notEnoughSlots(pts)) {
  885. this.dumpJitterBufInfo("Fatal: decoder buf is full", pts);
  886. //dropIncoming
  887. untilIDR = true;
  888. this.flushBuffer(untilIDR);
  889. if (this.notEnoughSlots(pts)) {
  890. untilIDR = false;
  891. this.flushBuffer(untilIDR);
  892. }
  893. pktDrop = this.handleNewPktOnFlush(data.isIDR, media.byteLength);
  894. if (pktDrop) return;
  895. }
  896. }
  897. mediaArray[index] = {
  898. pts: pts,
  899. receive_ts: Date.now(),
  900. decode_ts: 0,
  901. yuv_ts: 0,
  902. render_ts: 0,
  903. media: media,
  904. meta: meta,
  905. metadata: data.metadata,
  906. isIDR: data.isIDR,
  907. };
  908. if (data.isIDR == true) {
  909. this.bufferIFrame += 1;
  910. }
  911. this.framesReceivedBetweenTimerInterval += 1;
  912. if (
  913. this.framesReceivedBetweenTimerInterval >
  914. this.maxFramesReceivedBetweenTimerInterval
  915. ) {
  916. this.maxFramesReceivedBetweenTimerInterval =
  917. this.framesReceivedBetweenTimerInterval;
  918. }
  919. if (!this.aheadof(pts, this.prevSeq)) {
  920. // writePtr += (cur - prev)
  921. this.writePtr = this.seqAdd(
  922. this.writePtr,
  923. this.seqDiff(pts, this.prevSeq, CACHE_BUF_LENGTH),
  924. CACHE_BUF_LENGTH
  925. );
  926. if (this.seqAdd(index, 1, CACHE_BUF_LENGTH) != this.writePtr) {
  927. this.dumpJitterBufInfo(
  928. "dec worker internal info: index (" +
  929. index +
  930. ") != write_ptr (" +
  931. this.writePtr +
  932. ")"
  933. );
  934. // debugger
  935. }
  936. this.prevSeq = pts;
  937. }
  938. this.dumpJitterBufInfo("Leaving Decoder.prototype.receiveFrame");
  939. //refactor end
  940. };
  941. Decoder.prototype.startEmiter = function () {
  942. self.decoder.startEmit = true;
  943. if (cachedFirstFrame != undefined) {
  944. self.postMessage(cachedFirstFrame, [cachedFirstFrame.data.buffer]);
  945. console.log("cachedFirstFrame", cachedFirstFrame);
  946. send_out_buffer += 1;
  947. this.receivedEmit++;
  948. cachedFirstFrame = undefined;
  949. }
  950. if (cachedPanoramaFirstFrame != undefined) {
  951. self.postMessage(cachedPanoramaFirstFrame);
  952. send_out_buffer += 1;
  953. this.receivedEmit++;
  954. cachedPanoramaFirstFrame = undefined;
  955. }
  956. };
  957. /*
  958. Decoder.prototype.decodePanorama = function (data) {
  959. console.log("upload pano data");
  960. var content = data.data.data;
  961. var content_size = data.data.mediaLen;
  962. // var cacheBuffer = Module._malloc(content_size)
  963. // var resultBuffer = Module._malloc(64)
  964. Module.HEAPU8.set(content, cacheBuffer);
  965. let ret = 0;
  966. try {
  967. ret = Module._decodeData(0, 0, cacheBuffer, content_size, resultBuffer);
  968. // // console.log('[][Core][WASM] return value %s',ret)
  969. // if(ret!=0){
  970. // // console.log('[][Core][WASM],-abcdefg-----> ', ret)
  971. // var ret_close = Module._closeDecoder(0)
  972. // // eslint-disable-next-line no-empty
  973. // if (ret_close === 0) {
  974. // // console.log('[][Core][WASM] decoder closed for restart')
  975. // } else {
  976. // printConsole.error('close decoder failed after decode pano.')
  977. // return 1
  978. // }
  979. // var ret0 = Module._openDecoder(0, 0, 2)
  980. // // console.log('[][Core][WASM] decoder restart success')
  981. // // var ret1 = Module._openDecoder(1, decoder_type, LOG_LEVEL_WASM)
  982. // if (ret0 === 0) {
  983. // ret = Module._decodeData(0, 0, cacheBuffer, content_size, resultBuffer)
  984. // } else {
  985. // printConsole.error('openDecoder failed with error ' + String(ret0) , '5001')
  986. // return 1
  987. // }
  988. // }
  989. } catch (e) {
  990. console.log("catch error ", e);
  991. printConsole.error(e.message, "5002");
  992. }
  993. // let ret = Module._decodeData(0, 0, cacheBuffer, content_size, resultBuffer)
  994. var width = Module.getValue(resultBuffer, "i32");
  995. var height = Module.getValue(resultBuffer + 4, "i32");
  996. var stride_y = Module.getValue(resultBuffer + 20, "i32");
  997. var stride_u = Module.getValue(resultBuffer + 24, "i32");
  998. var stride_v = Module.getValue(resultBuffer + 28, "i32");
  999. var addr_y = Module.getValue(resultBuffer + 8, "i32");
  1000. var addr_u = Module.getValue(resultBuffer + 12, "i32");
  1001. var addr_v = Module.getValue(resultBuffer + 16, "i32");
  1002. var poc = Module.getValue(resultBuffer + 32, "i32");
  1003. if (ret != 0) {
  1004. printConsole.log(
  1005. "Decode Data error for panorama, ret value is " +
  1006. String(ret) +
  1007. ", frame content size: " +
  1008. String(content_size)
  1009. );
  1010. return;
  1011. }
  1012. var yuv_data = new Uint8Array((width * height * 3) / 2);
  1013. let pos = 0;
  1014. for (let i = 0; i < height; i++) {
  1015. let src = addr_y + i * stride_y;
  1016. let tmp = HEAPU8.subarray(src, src + width);
  1017. tmp = new Uint8Array(tmp);
  1018. yuv_data.set(tmp, pos);
  1019. pos += tmp.length;
  1020. }
  1021. for (let i = 0; i < height / 2; i++) {
  1022. let src = addr_u + i * stride_u;
  1023. let tmp = HEAPU8.subarray(src, src + width / 2);
  1024. tmp = new Uint8Array(tmp);
  1025. yuv_data.set(tmp, pos);
  1026. pos += tmp.length;
  1027. let src2 = addr_v + i * stride_v;
  1028. let tmp2 = HEAPU8.subarray(src2, src2 + width / 2);
  1029. tmp2 = new Uint8Array(tmp2);
  1030. yuv_data.set(tmp2, pos);
  1031. pos += tmp2.length;
  1032. }
  1033. const objData = {
  1034. t: MessageEvent.PanoramaMessage,
  1035. tileId: data.data.tileId,
  1036. uuid: data.data.uuid,
  1037. data: yuv_data,
  1038. x: data.data.x,
  1039. y: data.data.y,
  1040. z: data.data.z,
  1041. };
  1042. //TODO: remove debug
  1043. if (this.startEmit) {
  1044. self.postMessage(objData);
  1045. } else {
  1046. cachedPanoramaFirstFrame = objData;
  1047. }
  1048. // console.log('upload pano data with dataLength:', len(yuv_data))
  1049. var ret_close = Module._closeDecoder(0);
  1050. // eslint-disable-next-line no-empty
  1051. if (ret_close === 0) {
  1052. // console.log('[][Core][WASM] decoder closed for restart')
  1053. } else {
  1054. printConsole.error("close decoder failed after decode pano.");
  1055. return 1;
  1056. }
  1057. var ret0 = Module._openDecoder(0, 0, 2);
  1058. // var ret1 = Module._openDecoder(1, decoder_type, LOG_LEVEL_WASM)
  1059. if (ret0 === 0) {
  1060. // console.log('[][Core][WASM] decoder restart success')
  1061. self.decoder.startDecoding();
  1062. self.postMessage({
  1063. t: MessageEvent.WASMReady,
  1064. wasm_ready: true,
  1065. updateStats: false,
  1066. });
  1067. } else {
  1068. printConsole.error(
  1069. "openDecoder failed with error " + String(ret0),
  1070. "5001"
  1071. );
  1072. return 1;
  1073. }
  1074. };
  1075. */
  1076. Decoder.prototype.decodeFrame = function (data) {
  1077. var content = data.media;
  1078. if (typeof content == "undefined") {
  1079. printConsole.error("null content in decoder", "5999");
  1080. return;
  1081. }
  1082. var content_size = content.byteLength;
  1083. // var cacheBuffer = Module._malloc(content_size)
  1084. // var resultBuffer = Module._malloc(64)
  1085. Module.HEAPU8.set(content, cacheBuffer);
  1086. const index = data.frameCnt % CACHE_BUF_LENGTH;
  1087. mediaArray[index].decode_ts = Date.now();
  1088. var objData;
  1089. // debugger
  1090. if (content_size != 0) {
  1091. // var date = Date.now()
  1092. // var curDate = Date.now()
  1093. // while (curDate - date < 100) {
  1094. // curDate = Date.now()
  1095. // }
  1096. // TODO: Enable/Disable it by config
  1097. if (data.isIDR) {
  1098. this.errorCacheSize = 0;
  1099. }
  1100. // Guarantee that stream start from I frame
  1101. if (this.errorCacheSize != 0 || data.isIDR) {
  1102. this.errorCacheBuffer.set(content, this.mediaCacheSize);
  1103. this.errorCacheSize += content.byteLength;
  1104. }
  1105. let start_ts = Date.now();
  1106. let ret = 0;
  1107. try {
  1108. console.log('metaverse----WASM:cacheBuffer:'+cacheBuffer+',content_size:'+content_size+',resultBuffer:'+resultBuffer);
  1109. ret = Module._decodeData(
  1110. 0,
  1111. data.frameCnt,
  1112. cacheBuffer,
  1113. content_size,
  1114. resultBuffer
  1115. );
  1116. // if(ret==8){
  1117. // // console.log('[][Core][WASM],-abcdefg-----> ', ret)
  1118. // var ret_close = Module._closeDecoder(0)
  1119. // // eslint-disable-next-line no-empty
  1120. // if (ret_close === 0) {
  1121. // // console.log('[][Core][WASM] decoder closed for restart')
  1122. // } else {
  1123. // printConsole.error('close decoder failed after decode pano.')
  1124. // return 1
  1125. // }
  1126. // var ret0 = Module._openDecoder(0, 0, 2)
  1127. // // console.log('[][Core][WASM] decoder restart success')
  1128. // // var ret1 = Module._openDecoder(1, decoder_type, LOG_LEVEL_WASM)
  1129. // if (ret0 === 0) {
  1130. // ret = Module._decodeData(0, data.frameCnt, cacheBuffer, content_size, resultBuffer)
  1131. // } else {
  1132. // printConsole.error('openDecoder failed with error ' + String(ret0) , '5001')
  1133. // return 1
  1134. // }
  1135. // }
  1136. } catch (e) {
  1137. console.log("catch error ", e);
  1138. if (this.errorCacheSize > 0) {
  1139. downloadBlob(
  1140. this.errorCacheBuffer.subarray(0, this.errorCacheSize),
  1141. "error.264",
  1142. "application/octet-stream"
  1143. );
  1144. this.errorCacheSize = 0;
  1145. }
  1146. printConsole.error(e.message, "5002");
  1147. }
  1148. // core 420 to YUV
  1149. var width = Module.getValue(resultBuffer, "i32");
  1150. var height = Module.getValue(resultBuffer + 4, "i32");
  1151. var stride_y = Module.getValue(resultBuffer + 20, "i32");
  1152. var stride_u = Module.getValue(resultBuffer + 24, "i32");
  1153. var stride_v = Module.getValue(resultBuffer + 28, "i32");
  1154. var addr_y = Module.getValue(resultBuffer + 8, "i32");
  1155. var addr_u = Module.getValue(resultBuffer + 12, "i32");
  1156. var addr_v = Module.getValue(resultBuffer + 16, "i32");
  1157. var poc = Module.getValue(resultBuffer + 32, "i32");
  1158. var pts = data.frameCnt;
  1159. console.log("ret", ret);
  1160. if (ret != 0) {
  1161. printConsole.log(
  1162. "Decode Data error for video stream, ret value is " +
  1163. String(ret) +
  1164. ", frame content size: " +
  1165. String(content_size)
  1166. );
  1167. if (this.errorCacheSize > 0) {
  1168. downloadBlob(
  1169. this.errorCacheBuffer.subarray(0, this.errorCacheSize),
  1170. "error.264",
  1171. "application/octet-stream"
  1172. );
  1173. this.errorCacheSize = 0;
  1174. }
  1175. printConsole.log(
  1176. "current poc is " + String(poc) + ", last poc is " + String(lastPoc)
  1177. );
  1178. return;
  1179. }
  1180. lastPoc = poc;
  1181. this.receivedYUV++;
  1182. let end_ts = Date.now();
  1183. fdt = end_ts - start_ts;
  1184. if (fdt + self.decoder.getFrameInterval > 84) {
  1185. this.JankTimes++;
  1186. }
  1187. if (fdt + self.decoder.getFrameInterval > 125) {
  1188. this.bigJankTimes++;
  1189. }
  1190. self.decoder.dtpf = self.decoder.dtpf * 0.9 + fdt * 0.1;
  1191. // if (fdt > self.decoder.dtmf) {
  1192. // self.decoder.dtmf = fdt
  1193. // }
  1194. self.decoder.decodeTimeCircular[self.decoder.dtcPtr] = fdt;
  1195. self.decoder.dtcPtr =
  1196. (self.decoder.dtcPtr + 1) % self.decoder.decodeTimeCircular.length;
  1197. if (YUVArray.length <= 0) {
  1198. // printConsole.error('No buffer to save YUV after decoding, pts is ' + String(pts), '5002')
  1199. return;
  1200. }
  1201. var first_available_buffer = YUVArray.shift();
  1202. var yuv_data = first_available_buffer.buffer;
  1203. let pos = 0;
  1204. for (let i = 0; i < height; i++) {
  1205. let src = addr_y + i * stride_y;
  1206. let tmp = HEAPU8.subarray(src, src + width);
  1207. tmp = new Uint8Array(tmp);
  1208. yuv_data.set(tmp, pos);
  1209. pos += tmp.length;
  1210. }
  1211. for (let i = 0; i < height / 2; i++) {
  1212. let src = addr_u + i * stride_u;
  1213. let tmp = HEAPU8.subarray(src, src + width / 2);
  1214. tmp = new Uint8Array(tmp);
  1215. yuv_data.set(tmp, pos);
  1216. pos += tmp.length;
  1217. let src2 = addr_v + i * stride_v;
  1218. let tmp2 = HEAPU8.subarray(src2, src2 + width / 2);
  1219. tmp2 = new Uint8Array(tmp2);
  1220. yuv_data.set(tmp2, pos);
  1221. pos += tmp2.length;
  1222. }
  1223. objData = {
  1224. t: MessageEvent.DecodeMessage,
  1225. data: yuv_data,
  1226. width: width,
  1227. height: height,
  1228. pts: data.frameCnt,
  1229. yuv_ts: Date.now(),
  1230. meta: data.meta,
  1231. metadata: data.metadata,
  1232. };
  1233. } else {
  1234. objData = {
  1235. t: MessageEvent.DecodeMessage,
  1236. data: null,
  1237. width: 0,
  1238. height: 0,
  1239. pts: data.frameCnt,
  1240. yuv_ts: Date.now(),
  1241. meta: data.meta,
  1242. metadata: data.metadata,
  1243. };
  1244. }
  1245. if (this.startEmit) {
  1246. if (objData.data != null) {
  1247. // console.log("hey", pts, objData, objData.data);
  1248. self.postMessage(objData, [objData.data.buffer]);
  1249. send_out_buffer += 1;
  1250. this.receivedEmit++;
  1251. } else {
  1252. self.postMessage(objData);
  1253. this.receivedEmit++;
  1254. }
  1255. } else {
  1256. if (objData.data != null) {
  1257. cachedFirstFrame = objData;
  1258. }
  1259. }
  1260. console.log("this.startEmit", this.startEmit);
  1261. // if (cacheBuffer != null) {
  1262. // Module._free(cacheBuffer)
  1263. // cacheBuffer = null
  1264. // }
  1265. // if (resultBuffer != null) {
  1266. // Module._free(resultBuffer)
  1267. // resultBuffer = null
  1268. // }
  1269. return;
  1270. };
  1271. Decoder.prototype.receiveBuffer = function (data) {
  1272. framesReturned++;
  1273. send_out_buffer -= 1;
  1274. console.log("receiveBuffer", data);
  1275. YUVArray.push({ status: 0, buffer: data.buffer });
  1276. };
  1277. Decoder.prototype.setPassiveJitter = function (len) {
  1278. this.passiveJitterLength = len;
  1279. };
  1280. Decoder.prototype.uninitDecoder = function () {
  1281. printConsole.log("Going to uninit decoder.");
  1282. };
  1283. Decoder.prototype.StartRecord = function () {
  1284. printConsole.log("Start Record");
  1285. this.startRecord = true;
  1286. };
  1287. Decoder.prototype.SaveRecord = function () {
  1288. printConsole.log("Save Record");
  1289. this.saveRecord = true;
  1290. };
  1291. Decoder.prototype.ReceivePanorama = function (data) {
  1292. self.decoder.resetDecoder();
  1293. self.decoder.decodePanorama(data);
  1294. };
  1295. Decoder.prototype.LoadWASM = function (url) {
  1296. printConsole.error("Load WASM from " + String(url));
  1297. try {
  1298. self.importScripts(url);
  1299. } catch (e) {
  1300. console.log("catch error ", e);
  1301. printConsole.error(e.message, "5003");
  1302. }
  1303. };
  1304. // self.incoming_pkt_queue = new array()
  1305. function getRandomInt(max) {
  1306. return Math.floor(Math.random() * max);
  1307. }
  1308. // console.log(getRandomInt(30));
  1309. self.decoder = new Decoder();
  1310. netArray = [];
  1311. var gTmpIdx = 0;
  1312. var gLossCnt = 0;
  1313. self.onmessage = function (evt) {
  1314. switch (evt.data.t) {
  1315. case 1: // Init Message
  1316. self.decoder.initAll(evt.data.config);
  1317. break;
  1318. case 0: // Decode Message
  1319. // console.log('[][Core][WASM],------> ', evt.data)
  1320. gTmpIdx += 1;
  1321. randLen = 16;
  1322. // randLen = getRandomInt(30)
  1323. // eslint-disable-next-line no-constant-condition
  1324. if (gTmpIdx > 100 && false) {
  1325. var test_jitter_buffer = true;
  1326. if (test_jitter_buffer == true) {
  1327. if (netArray.length % 5 == 4) {
  1328. // netArray.insert(netArray.length -1, evt.data)
  1329. netArray.splice(netArray.length - 1, 0, evt.data);
  1330. } else {
  1331. netArray.push(evt.data);
  1332. }
  1333. if (netArray.length > randLen) {
  1334. // 1. jitter
  1335. while (netArray.length > 0) {
  1336. // console.log("[xmedia] array len: %s", netArray.length)
  1337. gLossCnt += 1;
  1338. var pkt = netArray.shift();
  1339. // lose pkt
  1340. var dropInterval = 50;
  1341. var dropContinousPkts = 3;
  1342. if (gLossCnt % dropInterval < dropContinousPkts) {
  1343. if (gLossCnt == dropInterval + dropContinousPkts - 1) {
  1344. gLossCnt = 0;
  1345. }
  1346. } else {
  1347. self.decoder.receiveFrame(pkt);
  1348. }
  1349. }
  1350. // // 2. disorder
  1351. // if (incoming_pkt_queue.length % 3) {
  1352. // in[0]
  1353. // in[2]
  1354. // in[1]
  1355. // }
  1356. }
  1357. } else {
  1358. self.decoder.receiveFrame(evt.data);
  1359. }
  1360. } else {
  1361. self.decoder.receiveFrame(evt.data);
  1362. }
  1363. break;
  1364. case 2: // Receive used buffer
  1365. self.decoder.receiveBuffer(evt.data);
  1366. break;
  1367. case 3: // Unint Message
  1368. self.decoder.uninitDecoder();
  1369. break;
  1370. case 4: // Reset status
  1371. self.decoder.resetDecoder();
  1372. break;
  1373. case 5: // Start emit
  1374. // debugger
  1375. self.decoder.startEmiter();
  1376. break;
  1377. case 6: // Start Record
  1378. self.decoder.StartRecord();
  1379. break;
  1380. case 7: // Save Record
  1381. self.decoder.SaveRecord();
  1382. break;
  1383. case 8: // Panorama Decode Message
  1384. self.decoder.ReceivePanorama(evt.data);
  1385. break;
  1386. case 9: // Select WASM Version
  1387. self.decoder.setPassiveJitter(evt.data.jitterLength);
  1388. self.decoder.LoadWASM(evt.data.url);
  1389. break;
  1390. case 100: // change decoder worker status
  1391. self.decoder.changeLogSwitch(evt.data.status);
  1392. break;
  1393. }
  1394. };
  1395. }