update

src/move/move.service.ts

@@ -199,24 +199,43 @@ export class MoveService implements OnModuleInit {
     this.users[userId] = user;
   }
 
-  async getMoveFrames(appId,start_break_point_id,end_break_point_id,angleIndex){
-
-    let moveFramesRes,moveFrames;
-    let key = 'moveframe:app_id:' + appId + ':start_break_point_id:' + start_break_point_id + ':end_break_point_id:' + end_break_point_id +':angle:' + angleIndex;
+  async getMoveFrames(
+    appId,
+    start_break_point_id,
+    end_break_point_id,
+    angleIndex,
+  ) {
+    let moveFramesRes, moveFrames;
+    let key =
+      'moveframe:app_id:' +
+      appId +
+      ':start_break_point_id:' +
+      start_break_point_id +
+      ':end_break_point_id:' +
+      end_break_point_id +
+      ':angle:' +
+      angleIndex;
     //倒叙
-    if(start_break_point_id>end_break_point_id){
-      key = 'moveframe:app_id:' + appId + ':start_break_point_id:' + end_break_point_id + ':end_break_point_id:' + start_break_point_id +':angle:' + angleIndex;
-      console.log('getMoveFrames:'+key);
+    if (start_break_point_id > end_break_point_id) {
+      key =
+        'moveframe:app_id:' +
+        appId +
+        ':start_break_point_id:' +
+        end_break_point_id +
+        ':end_break_point_id:' +
+        start_break_point_id +
+        ':angle:' +
+        angleIndex;
+      console.log('getMoveFrames:' + key);
       moveFramesRes = await this.cacheService.get(key);
       moveFrames = JSON.parse(moveFramesRes);
       moveFrames = moveFrames.reverse();
-    }
-    else{
-      console.log('getMoveFrames:'+key);
+    } else {
+      console.log('getMoveFrames:' + key);
       moveFramesRes = await this.cacheService.get(key);
       moveFrames = JSON.parse(moveFramesRes);
     }
-    return moveFrames
+    return moveFrames;
   }
 
   async move(pathArray, actionRequest) {
@@ -249,7 +268,7 @@ export class MoveService implements OnModuleInit {
       this.reply.traceIds = traceIds;
       this.reply['newUserStates'][0].userId = userId;
       this.reply['actionResponses'][0].traceId = traceId;
-      const index = Math.floor((user.camera.angle.yaw + 1) / 45)%8; //过渡需要
+      const index = Math.floor((user.camera.angle.yaw + 1) / 45) % 8; //过渡需要
       for (let i = 0; i < path.length - 1; ++i) {
         let pathReplys = [];
         const start_break_point_id = path[i];
@@ -271,7 +290,12 @@ export class MoveService implements OnModuleInit {
         }
         const moveFrames = JSON.parse(moveFramesRes);
         */
-        const moveFrames = await this.getMoveFrames(appId,start_break_point_id,end_break_point_id,index);
+        const moveFrames = await this.getMoveFrames(
+          appId,
+          start_break_point_id,
+          end_break_point_id,
+          index,
+        );
         if (!moveFrames) {
           return replys;
         }
@@ -388,18 +412,18 @@ export class MoveService implements OnModuleInit {
         moveFrame.file_name +
         '?m=' +
         new Date().getTime();
-      if(startBreakPointId>endBreakPointId){
+      if (startBreakPointId > endBreakPointId) {
         reply.mediaSrc =
-        '/' +
-        appId +
-        '/' +
-        endBreakPointId +
-        '/' +
-        moveFrame.file_name.substring(0, moveFrame.file_name.indexOf('.')) +
-        '/' +
-        moveFrame.file_name +
-        '?m=' +
-        new Date().getTime();
+          '/' +
+          appId +
+          '/' +
+          endBreakPointId +
+          '/' +
+          moveFrame.file_name.substring(0, moveFrame.file_name.indexOf('.')) +
+          '/' +
+          moveFrame.file_name +
+          '?m=' +
+          new Date().getTime();
       }
 
       reply.startBreakPointId = startBreakPointId;
@@ -465,18 +489,18 @@ export class MoveService implements OnModuleInit {
         '?m=' +
         new Date().getTime();
 
-      if(startBreakPointId>endBreakPointId){
+      if (startBreakPointId > endBreakPointId) {
         reply.mediaSrc =
-        '/' +
-        appId +
-        '/' +
-        endBreakPointId +
-        '/' +
-        moveFrame.file_name.substring(0, moveFrame.file_name.indexOf('.')) +
-        '/' +
-        moveFrame.file_name +
-        '?m=' +
-        new Date().getTime();
+          '/' +
+          appId +
+          '/' +
+          endBreakPointId +
+          '/' +
+          moveFrame.file_name.substring(0, moveFrame.file_name.indexOf('.')) +
+          '/' +
+          moveFrame.file_name +
+          '?m=' +
+          new Date().getTime();
       }
       reply.startBreakPointId = startBreakPointId;
       reply.endBreakPointId = endBreakPointId;
@@ -937,7 +961,7 @@ export class MoveService implements OnModuleInit {
         neighPoint.angle = angle;
         neighPoints.push(neighPoint);
         ++count;
-        console.log('handlejoysticktesttest:45°的倍数')
+        console.log('handlejoysticktesttest:45°的倍数');
         //debugger;
         break;
 
@@ -1218,9 +1242,14 @@ export class MoveService implements OnModuleInit {
       const moveFramesRes = await this.cacheService.get(key);
       const moveFrames = JSON.parse(moveFramesRes);
       */
-      const moveFrames = await this.getMoveFrames(appId,breakPointId,chooseBreakPointId,Math.floor(user.camera.angle.yaw / 45))
-      this.setCameraInfo(appId,moveFrames,breakPointId,chooseBreakPointId);
-      
+      const moveFrames = await this.getMoveFrames(
+        appId,
+        breakPointId,
+        chooseBreakPointId,
+        Math.floor(user.camera.angle.yaw / 45),
+      );
+      this.setCameraInfo(appId, moveFrames, breakPointId, chooseBreakPointId);
+
       user.breakPointId = chooseBreakPointId;
 
       const cameraInfo = this.getCameraInfo();
@@ -1263,21 +1292,21 @@ export class MoveService implements OnModuleInit {
         '?m=' +
         new Date().getTime();
 
-      if(startBreakPointId>endBreakPointId){
+      if (startBreakPointId > endBreakPointId) {
         moveFrames[i].mediaSrc =
-        '/' +
-        appId +
-        '/' +
-        endBreakPointId +
-        '/' +
-        moveFrames[i].file_name.substring(
-          0,
-          moveFrames[i].file_name.indexOf('.'),
-        ) +
-        '/' +
-        moveFrames[i].file_name +
-        '?m=' +
-        new Date().getTime();
+          '/' +
+          appId +
+          '/' +
+          endBreakPointId +
+          '/' +
+          moveFrames[i].file_name.substring(
+            0,
+            moveFrames[i].file_name.indexOf('.'),
+          ) +
+          '/' +
+          moveFrames[i].file_name +
+          '?m=' +
+          new Date().getTime();
       }
       this.cameraInfos.push(moveFrames[i]);
     }
@@ -1339,12 +1368,13 @@ export class MoveService implements OnModuleInit {
     }
   }
 
-  stopJoystick(userId){
+  stopJoystick(userId) {
     this.reply.traceIds = [];
     const user = this.users[userId];
     this.reply['newUserStates'][0].userId = userId;
     this.reply['actionResponses'][0].traceId = null;
-    this.reply['newUserStates'][0].playerState.player.angle.yaw = user.player.angle.yaw;
+    this.reply['newUserStates'][0].playerState.player.angle.yaw =
+      user.player.angle.yaw;
 
     this.reply['newUserStates'][0].playerState.player.position = JSON.parse(
       JSON.stringify(user.player.position),

src/scene/scene.module.ts

@@ -7,9 +7,7 @@ import { RotateService } from '../rotate/rotate.service';
 import { MoveService } from '../move/move.service';
 import { GetRouterService } from 'src/get-router/get-router.service';
 @Module({
-  imports: [
-    CacheModule
-  ],
+  imports: [CacheModule],
   controllers: [],
   providers: [
     SceneService,

src/scene/scene.service.ts

@@ -26,7 +26,7 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
     private rotateService: RotateService,
     private moveService: MoveService,
     private getRouterService: GetRouterService, // @InjectQueue('rotate') private rotateQueue: Queue, // @InjectQueue('walking') private walkingQueue: Queue,
-  ) { }
+  ) {}
   @Client(grpcClientOptions) private readonly client: ClientGrpc;
 
   public _frameInteval: NodeJS.Timeout;
@@ -34,6 +34,7 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
   public _rotateTimeout: NodeJS.Timeout;
   public _moveTimeout: NodeJS.Timeout;
   public _JoyStickingTimeout: NodeJS.Timeout;
+  public _JoyStickingSteamTimeout: NodeJS.Timeout;
   public startSteaming = new BehaviorSubject<boolean>(false);
   public onRotating = new BehaviorSubject<boolean>(false);
   public onMoving = new BehaviorSubject<boolean>(false);
@@ -102,7 +103,6 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
 
   private isJoystickHasStream = false;
 
-
   public users = {};
 
   public sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
@@ -269,16 +269,16 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
         const playerAngle = newUserStates.playerState.player.angle;
         this.logger.log(
           'stop-data-0' +
-          'trace_id: ' +
-          trace_id +
-          'userId:' +
-          userId +
-          'breakPointId :' +
-          breakPointId +
-          'cameraAngle :' +
-          JSON.stringify(cameraAngle) +
-          'playerAngle: ' +
-          JSON.stringify(playerAngle),
+            'trace_id: ' +
+            trace_id +
+            'userId:' +
+            userId +
+            'breakPointId :' +
+            breakPointId +
+            'cameraAngle :' +
+            JSON.stringify(cameraAngle) +
+            'playerAngle: ' +
+            JSON.stringify(playerAngle),
         );
         //debugger;
         console.log('moveService.stop-1:' + breakPointId);
@@ -327,7 +327,7 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
             const rounded = Number(inMillSeconds).toFixed(3);
             this.logger.log(
               `[timer]-rotate-入队列前: ${rounded}ms -->` +
-              JSON.stringify(stream),
+                JSON.stringify(stream),
             );
 
             this.roQueue.next(stream);
@@ -431,15 +431,15 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
       console.log('进入1 - searchRoad');
       this.logger.log(
         'handleWalking-users' +
-        JSON.stringify(this.moveService.users) +
-        ' this.user_id: ' +
-        this.user_id,
+          JSON.stringify(this.moveService.users) +
+          ' this.user_id: ' +
+          this.user_id,
       );
       this.logger.log(
         'handleWalking-currentUser' +
-        JSON.stringify(user) +
-        ' this.user_id: ' +
-        this.user_id,
+          JSON.stringify(user) +
+          ' this.user_id: ' +
+          this.user_id,
       );
       console.log('path-start' + user.breakPointId);
 
@@ -509,16 +509,16 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
         if (seqs?.length) {
           this.logger.log(
             'walking --队列总览:' +
-            ' 总段数： ' +
-            walkingRes.length +
-            ' 镜头帧数：' +
-            walkingRes[0].length +
-            ' 行走段数：' +
-            (walkingRes[0]?.length
-              ? walkingRes.length - 1
-              : walkingRes.length) +
-            ' 队列总帧数：' +
-            seqs.length,
+              ' 总段数： ' +
+              walkingRes.length +
+              ' 镜头帧数：' +
+              walkingRes[0].length +
+              ' 行走段数：' +
+              (walkingRes[0]?.length
+                ? walkingRes.length - 1
+                : walkingRes.length) +
+              ' 队列总帧数：' +
+              seqs.length,
           );
           const stop = performance.now();
           const inMillSeconds = stop - start;
@@ -568,16 +568,86 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
    */
 
   async handlejoystick(request: JoystickRequest) {
+    // 二级方法start，为了 drain干complementFrame的pool
+    const pushJoyStickSteam = async (joystickRes: StreamReplyType) => {
+      this.holdSteam();
+      console.log('joystickRes有mediaSrc', joystickRes.mediaSrc);
+      const mediaSrc = joystickRes.mediaSrc.split('?')[0];
+      // IDR flag设置为I帧
+      const setDIR = joystickRes.moveOver || joystickRes.moveStart ? 1 : 3;
+      // 过滤新东西, 推完给回false
+      this.moveService.sendingFrameForJoystick = true;
+
+      if (this.joystickFrameCnt === -1) {
+        this.joystickFrameCnt = this.frameCnt.getValue();
+      }
+      this.joystickFrameCnt += 1;
+      const streamData: StreamFrameType = {
+        frame: this.joystickFrameCnt,
+        clipPath: mediaSrc,
+        metaData: JSON.stringify(joystickRes),
+        serverTime: this.mockserverTime,
+        DIR: setDIR,
+      };
+
+      const hasPush = await this.streamService.pushFrameToSteam(streamData);
+      if (hasPush.done) {
+        this.isJoystickHasStream = true;
+        console.log('joystick-hasPush', hasPush);
+        if (this.isJoystickHasStream) {
+          await this.sleep(30);
+        }
+        this.moveService.sendingFrameForJoystick = false;
+        const data = joystickRes as StreamReplyType;
+        console.log('handlejoystick-moveOver:' + data.moveOver);
+        if (data?.moveOver && data.moveOver) {
+          // moveOver
+          console.log('回传updateUser', data);
+          // const userId = this.user_id;
+          // 回传点暂时有问题，待修复
+          //const breakPointId = data.endBreakPointId || data.breakPointId;
+          //const lastReply = JSON.stringify(joystickRes);
+          //this.moveService.updateUser(userId, breakPointId, lastReply);
+        }
+        clearTimeout(this._JoyStickingSteamTimeout);
+        this._JoyStickingSteamTimeout = setTimeout(() => {
+          const complementFrame = this.moveService.complementFrame(
+            this.user_id,
+          ) as StreamReplyType;
+          console.log('has-complementFrame', complementFrame);
+          if (complementFrame) {
+            pushJoyStickSteam(complementFrame);
+          } else {
+            this.frameCnt.next(hasPush.frame);
+            this.logger.log('joystick opt done');
+            this.logger.log('joystick 交权给空流,当前pts', hasPush.frame);
+            // this.frameCnt.next(res.frame);
+            this.onJoysticking.next(false);
+            this.resumeStream();
+            this.joystickFrameCnt = -1;
+            this.isJoystickHasStream = false;
+          }
+        }, 200);
+      } else {
+        console.error('joystick-流地址有误::', joystickRes.mediaSrc);
+        this.logger.error('joystick-流地址有误::', joystickRes.mediaSrc);
+        this.resumeStream();
+      }
+
+      return hasPush;
+    };
+    // 二级方法end
+
     try {
       //const joystickRes = await this.moveService.joystick(request);
       this._rotateCount = 0;
       const joystickRes = await this.moveService.seqExeJoystick(request);
       this.logger.log(
         'joystick-breakPointId:' +
-        this.moveService.users[this.user_id].breakPointId,
+          this.moveService.users[this.user_id].breakPointId,
       );
       // 有数据 [0]是rotate数据，[1-infinity]是walking数据
-      this.logger.log('joystickRes-1', joystickRes);
+      this.logger.log('joystickRes', JSON.stringify(joystickRes));
 
       if (joystickRes) {
         this.onJoysticking.next(true);
@@ -590,76 +660,14 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
         //     ),
         console.log(
           'handlejoysticktesttest:' +
-          joystickRes.mediaSrc +
-          '，相机坐标：' +
-          JSON.stringify(
-            joystickRes.newUserStates[0].playerState.player.position,
-          ),
+            joystickRes.mediaSrc +
+            '，相机坐标：' +
+            JSON.stringify(
+              joystickRes.newUserStates[0].playerState.player.position,
+            ),
         );
         if (joystickRes.mediaSrc) {
-          this.holdSteam();
-          console.log('joystickRes有mediaSrc', joystickRes.mediaSrc);
-          const mediaSrc = joystickRes.mediaSrc.split('?')[0];
-          // IDR flag设置为I帧
-          const setDIR = joystickRes.moveOver || joystickRes.moveStart ? 1 : 3;
-          // 过滤新东西, 推完给回false
-          this.moveService.sendingFrameForJoystick = true;
-
-          if (this.joystickFrameCnt === -1) {
-            this.joystickFrameCnt = this.frameCnt.getValue();
-          }
-          this.joystickFrameCnt += 1;
-          const streamData: StreamFrameType = {
-            frame: this.joystickFrameCnt,
-            clipPath: mediaSrc,
-            metaData: JSON.stringify(joystickRes),
-            serverTime: this.mockserverTime,
-            DIR: setDIR,
-          };
-
-          const hasPush = await this.streamService.pushFrameToSteam(streamData);
-          if (hasPush.done) {
-            this.isJoystickHasStream = true;
-            console.log('joystick-hasPush', hasPush);
-            if (this.isJoystickHasStream) {
-              await this.sleep(30);
-            }
-            this.moveService.sendingFrameForJoystick = false;
-            const data = joystickRes as StreamReplyType;
-            console.log('handlejoystick-moveOver:' + data.moveOver);
-            if (data?.moveOver && data.moveOver) {
-              // moveOver
-              console.log('回传updateUser', data);
-              // const userId = this.user_id;
-              // 回传点暂时有问题，待修复
-              //const breakPointId = data.endBreakPointId || data.breakPointId;
-              //const lastReply = JSON.stringify(joystickRes);
-              //this.moveService.updateUser(userId, breakPointId, lastReply);
-            }
-            clearTimeout(this._JoyStickingTimeout);
-            this._JoyStickingTimeout = setTimeout(() => {
-              this.frameCnt.next(hasPush.frame);
-              this.logger.log('joystick opt done');
-              this.logger.log('joystick 交权给空流,当前pts', hasPush.frame);
-              // this.frameCnt.next(res.frame);
-              this.onJoysticking.next(false);
-              this.resumeStream();
-              this.joystickFrameCnt = -1;
-              this.isJoystickHasStream = false;
-            }, 200);
-          } else {
-            console.error(
-              'joystick-流地址有误::',
-              hasPush.frame,
-              joystickRes.mediaSrc,
-            );
-            this.logger.error(
-              'joystick-流地址有误::',
-              hasPush.frame,
-              joystickRes.mediaSrc,
-            );
-            this.resumeStream();
-          }
+          pushJoyStickSteam(joystickRes);
         } else {
           this.logger.log('joystick-接收人物数据', this.onMoving.getValue());
           if (!this.onMoving.getValue()) {
@@ -716,8 +724,6 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
     }
   }
 
-
-
   /**
    * 主要处理moving的序列动作
    * @param seqs StreamReplyType[]
@@ -791,14 +797,14 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
           };
           this.logger.log(
             '[media-move]: ' +
-            ', moveframeCnt: ' +
-            this.moveframeCnt +
-            ', clipPath: ' +
-            stream.clipPath +
-            ', mType: ' +
-            stream.mType +
-            ', DIR: ' +
-            stream.DIR,
+              ', moveframeCnt: ' +
+              this.moveframeCnt +
+              ', clipPath: ' +
+              stream.clipPath +
+              ', mType: ' +
+              stream.mType +
+              ', DIR: ' +
+              stream.DIR,
             // stream.metaData,
           );
           this.logger.log(
@@ -994,7 +1000,8 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
 
           const IDRflag = this._rotateCurrentFame % 5 === 0 ? 1 : 3;
           this.logger.log(
-            `当前rotate ,mainframeCnt:${this.frameCnt.getValue()}, _rotateCurrentFame:${this._rotateCurrentFame
+            `当前rotate ,mainframeCnt:${this.frameCnt.getValue()}, _rotateCurrentFame:${
+              this._rotateCurrentFame
             } IDRflag:${IDRflag}`,
           );
           stream.DIR = this.rotateFirstIDR ? 1 : IDRflag;
@@ -1005,12 +1012,12 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
 
           this.logger.log(
             '[media-rotate]: ' +
-            ', frame: ' +
-            stream.frame +
-            ', rotateframeCnt: ' +
-            this.rotateframeCnt +
-            ', clipPath: ' +
-            stream.clipPath,
+              ', frame: ' +
+              stream.frame +
+              ', rotateframeCnt: ' +
+              this.rotateframeCnt +
+              ', clipPath: ' +
+              stream.clipPath,
             // stream.metaData,
           );
           // this.logger.log(
@@ -1072,12 +1079,12 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
           this.logger.log('frame', frame);
           console.log(
             'mock' +
-            ' maxMessageSize: ' +
-            this.channel.maxMessageSize() +
-            ' bytesReceived: ' +
-            this.peer.bytesReceived() +
-            ' bytesSent: ' +
-            this.peer.bytesSent(),
+              ' maxMessageSize: ' +
+              this.channel.maxMessageSize() +
+              ' bytesReceived: ' +
+              this.peer.bytesReceived() +
+              ' bytesSent: ' +
+              this.peer.bytesSent(),
           );
           if (frame === 1) {
             redisData = await this.rotateService.echo(this.user_id, true);
@@ -1092,7 +1099,7 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
                 delete redisData.mediaSrc;
                 this.logger.log(
                   `user:${this.user_id}:first render stream` +
-                  JSON.stringify({ path: clipPath, meta: redisData }),
+                    JSON.stringify({ path: clipPath, meta: redisData }),
                 );
                 const status = await this.pushFirstRender(
                   clipPath,
@@ -1120,15 +1127,15 @@ export class SceneService implements OnModuleInit, OnModuleDestroy {
 
             console.log(
               '空白流条件-->:' +
-              isOk +
-              ' onMoving: ' +
-              this.onMoving.value +
-              ' onRotating: ' +
-              this.onRotating.value +
-              ' onJoysticking: ' +
-              this.onJoysticking.value +
-              ' firstRender: ' +
-              this.firstRender,
+                isOk +
+                ' onMoving: ' +
+                this.onMoving.value +
+                ' onRotating: ' +
+                this.onRotating.value +
+                ' onJoysticking: ' +
+                this.onJoysticking.value +
+                ' firstRender: ' +
+                this.firstRender,
             );
           }