unity 接收拼接数据进行纹理替换且保存相机纹理到rtsp server(一)

1 rtsp 协议后编码解码

rtsp协议的问题就是，拼接完成后，还需要编码，而unity里面再需要解码，需要的过程多了一步编码再解码，大大加重了

2 rtsp 协议后轻量编码

rtsp协议使用mjpeg进行图片传输。why？这样做的好处是解码端进行像素处理以后不用再进行h264和h265编码，而unity端也不用再解码一次，这样增加了程序运行效率

3 server

1 rtsp server
2 websocket server

c# c++ 都可以做websocket server，
使用c++ 做一个server
server 既能接收推流，又能接收流

4 unity client

unity 上建立一个两个plane，调整位置

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;
//using NetM
public class test : MonoBehaviour
{// Start is called before the first frame updateprivate string msg = null;string[] file1Lines;void Start(){string path = Application.dataPath + "/rtsp.txt";path = path.Replace("/", "\\");//File静态类方式if (File.Exists(path)){Debug.Log("FileExists");file1Lines = File.ReadAllLines(path);foreach (string line in file1Lines){Debug.Log(line);}}else{Debug.Log("FileNotExists");File.CreateText(path);}//Texture2D m_resultTure = new Texture2D((int)widthSize, (int)heighSize, TextureFormat.RGB24, false);//Texture2D m_resultTure = GetComponent(texture1);//texture rigidbody2D = GetComponent(texture1);//rawImage = GetComponent<RawImage>();//rawImage.texture = texture;//NetManager.M_Instance.Connect("ws://127.0.0.1:8888");   //本机地址}private void OnGUI(){//绘制输入框，以及获取输入框中的内容//PS：第二参数必须是msg，否则在我们输入后，虽然msg可以获得到输入内容，但马上就被第二参数在下一帧重新覆盖。msg = GUI.TextField(new Rect(10, 10, 100, 20), msg);//绘制按钮，以及按下发送信息按钮，发送信息if (GUI.Button(new Rect(120, 10, 80, 20), "发送信息") && msg != null){NetManager.M_Instance.Send(msg);}//绘制按钮，以及按下断开连接按钮，发送断开连接请求if (GUI.Button(new Rect(210, 10, 80, 20), "断开连接")){Debug.Log("向服务器请求断开连接......");NetManager.M_Instance.CloseClientWebSocket();}}void Update(){}
}

4.1 websocket client

using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Net.WebSockets;
using System.Threading;
using System.Text;public class NetManager
{#region 实现单例的代码//变量private volatile static NetManager m_instance;          private static object m_locker = new object();          //属性public static NetManager M_Instance{get{//线程锁。防止同时判断为null时同时创建对象lock (m_locker){//如果不存在对象则创建对象if (m_instance == null){m_instance = new NetManager();}}return m_instance;}}#endregion//私有化构造private NetManager() { }//客户端webSocketprivate ClientWebSocket m_clientWebSocket;//处理接收数据的线程private Thread m_dataReceiveThread;//线程持续执行的标识符private bool m_isDoThread;private bool m_isConnected = false;/// <summary>/// ClientWebSocket，与服务器建立连接。/// </summary>/// <param name="uriStr"></param>public void Connect(string uriStr){try{//创建ClientWebSocketm_clientWebSocket = new ClientWebSocket();//初始化标识符m_isDoThread = true;//创建线程m_dataReceiveThread = new Thread(ReceiveData);  //创建数据接收线程  m_dataReceiveThread.IsBackground = true;        //设置为后台可以运行，主线程关闭时，此线程也会关闭（实际在Unity中并没什么用，还是要手动关闭）//设置请求头部//m_clientWebSocket.Options.SetRequestHeader("headerName", "hearValue");//开始连接var task = m_clientWebSocket.ConnectAsync(new Uri(uriStr), CancellationToken.None);task.Wait();    //等待//启动数据接收线程m_dataReceiveThread.Start(m_clientWebSocket);//输出提示if (m_clientWebSocket.State == WebSocketState.Open){Debug.Log("连接服务器完毕。");m_isConnected = true;}}catch (WebSocketException ex){Debug.LogError("连接出错：" + ex.Message);Debug.LogError("WebSokcet状态：" + m_clientWebSocket.State);CloseClientWebSocket();}}/// <summary>/// 持续接收服务器的信息。/// </summary>/// <param name="socket"></param>private void ReceiveData(object socket){//类型转换ClientWebSocket socketClient = (ClientWebSocket)socket;//持续接收信息while (m_isDoThread){//接收数据string data = Receive(socketClient);//数据处理（可以和服务器一样使用事件（委托）来处理）if (data != null){Debug.Log("接收的服务器消息：" + data);}//组装纹理//Texture2D tex2D = new Texture2D(100, 100);//tex2D.LoadImage(bytes);}Debug.Log("接收信息线程结束。");}/// <summary>/// 接收服务器信息。/// </summary>/// <param name="socket"></param>/// <returns></returns>private byte[] Receive(ClientWebSocket socket){try{//接收消息时，对WebSocketState是有要求的，所以加上if判断（如果不是这两种状态，会报出异常）if (socket != null && (socket.State == WebSocketState.Open || socket.State == WebSocketState.CloseSent)){byte[] arrry = new byte[1024];  //注意长度，如果服务器发送消息过长，这也需要跟着调整ArraySegment<byte> buffer = new ArraySegment<byte>(arrry);  //实例化一个ArraySegment结构体//接收数据var task = socket.ReceiveAsync(buffer, CancellationToken.None);task.Wait();//等待//仅作状态展示。在客户端发送关闭消息后，服务器会回复确认信息，在收到确认信息后状态便是CloseReceived，这里打印输出。Debug.Log("socekt当前状态：" + socket.State);//如果是结束消息确认，则返回null，不再解析信息if (socket.State == WebSocketState.CloseReceived || task.Result.MessageType == WebSocketMessageType.Close){return null;}//如果是字符串//return Encoding.UTF8.GetString(buffer.Array, 0, task.Result.Count);return buffer.Array;}else{return null;}}catch (WebSocketException ex){Debug.LogError("接收服务器信息错误：" + ex.Message);CloseClientWebSocket();return null;}}/// <summary>/// 发送消息/// </summary>/// <param name="content"></param>public void Send(string content){try{//发送消息时，对WebSocketState是有要求的，加上if判断（如果不是这两种状态，会报出异常）if (m_clientWebSocket != null && (m_clientWebSocket.State == WebSocketState.Open || m_clientWebSocket.State == WebSocketState.CloseReceived)){ArraySegment<byte> array = new ArraySegment<byte>(Encoding.UTF8.GetBytes(content)); //创建内容的字节编码数组并实例化一个ArraySegment结构体var task = m_clientWebSocket.SendAsync(array, WebSocketMessageType.Binary, true, CancellationToken.None);  //发送task.Wait();  //等待Debug.Log("发送了一个消息到服务器。");}}catch (WebSocketException ex){Debug.LogError("向服务器发送信息错误：" + ex.Message);CloseClientWebSocket();}}/// <summary>/// 关闭ClientWebSocket。/// </summary>public void CloseClientWebSocket(){//关闭Socketif (m_clientWebSocket != null && m_clientWebSocket.State == WebSocketState.Open){var task = m_clientWebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None);Debug.Log("socekt状态：" + m_clientWebSocket.State);Debug.Log("close");}//关闭线程if (m_dataReceiveThread != null && m_dataReceiveThread.IsAlive){m_isDoThread = false;   m_dataReceiveThread = null;}}
}

以上为websocket client， 接收到数据以后就可以替换纹理，接下去我们把里面的纹理图像保存出来，这样可以使用多个不同的camera 来保存不同角度和三维场景中不同的图像流。然后使用websocket 传输出去，形成一个rtsp server。

4.2 脚本start时候读取本地纹理

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;
using System;
public class canvasrtsp : MonoBehaviour
{// Start is called before the first frame update//public RawImage rawImage;//相机渲染的UI//public GameObject plane1;//相机渲染的GameObject//StartCoroutine(LoadTextureFromInternet("http://avatar.csdnimg.cn/1/E/6/2_u013012420.jpg"));//IEnumerator LoadTextureFromInternet(string path)//{//    UnityWebRequest request = new UnityWebRequest(path);//    //给request的downloadhandle赋值，new出来的UnityWebRequest不附加downloadhandle//    //数据，不赋值的话访问不到下载出来的数据//    DownloadHandlerTexture texture = new DownloadHandlerTexture(true);//    request.downloadHandler = texture;//    yield return request.Send();//    if (string.IsNullOrEmpty(request.error))//    {//        pic = texture.texture;//    }//    Image tempImage = GameObject.Find("Image").GetComponent<Image>();//    Sprite sp = Sprite.Create((Texture2D)pic, new Rect(0, 0, pic.width, pic.height), Vector2.zero);//    tempImage.sprite = sp;//    GameObject go = GameObject.Find("plane1");//    go.GetComponent<MeshRenderer>().material.mainTexture = pic;//}Texture2D Base64ToRGBA32(string imageData, int offset = 0){Texture2D tex2D = new Texture2D(2, 2, TextureFormat.RGB24, false);imageData = imageData.Substring(offset);byte[] data = Convert.FromBase64String(imageData);tex2D.LoadImage(data);return tex2D;}void Start(){string path1 = @"F:\pic\b.png";FileStream fileStream = new FileStream(path1, FileMode.Open, FileAccess.Read);fileStream.Seek(0, SeekOrigin.Begin);byte[] bye = new byte[fileStream.Length];fileStream.Read(bye, 0, bye.Length);fileStream.Close();//创建textureTexture2D texture2D = new Texture2D(240, 144);texture2D.LoadImage(bye);//plane1.gameObject.SetActive(true);GameObject go = GameObject.Find("Plane");//GameObject go1 = GameObject.FindWithTag("ppp");GameObject go1 = GameObject.Find("Cube");//Debug.Log(texture2D.width, texture2D.height);go.GetComponent<MeshRenderer>().material.mainTexture = texture2D;go1.GetComponent<MeshRenderer>().material.mainTexture = texture2D;//T image = GetComponent("image1");}// Update is called once per framevoid Update(){}
}

5 保存相机纹理

增加一个camera capture函数，这个函数将当前相机所见保存成图像

 Texture2D CaptureCamera(Camera camera, Rect rect){RenderTexture rt = new RenderTexture((int)rect.width, (int)rect.height, 0);//创建一个RenderTexture对象camera.targetTexture = rt;//临时设置相关相机的targetTexture为rt, 并手动渲染相关相机camera.Render();//ps: --- 如果这样加上第二个相机，可以实现只截图某几个指定的相机一起看到的图像。//ps: camera2.targetTexture = rt;//ps: camera2.Render();//ps: -------------------------------------------------------------------RenderTexture.active = rt;//激活这个rt, 并从中中读取像素。Texture2D screenShot = new Texture2D((int)rect.width, (int)rect.height, TextureFormat.RGB24, false);screenShot.ReadPixels(rect, 0, 0);//注：这个时候，它是从RenderTexture.active中读取像素screenShot.Apply();//重置相关参数，以使用camera继续在屏幕上显示camera.targetTexture = null;//ps: camera2.targetTexture = null;RenderTexture.active = null; //JC: added to avoid errorsGameObject.Destroy(rt);byte[] bytes = screenShot.EncodeToPNG();//最后将这些纹理数据，成一个png图片文件//string filename = Application.dataPath + "/Screenshot.png";string filename = "e:/qianbo2.png";System.IO.File.WriteAllBytes(filename, bytes);Debug.Log(string.Format("截屏了一张照片: {0}", filename));return screenShot;}

readpixel 方式

readpixel方式一定要注意等到 frameEnd 结束才能读取

WaitForEndOfFrame frameEnd = new WaitForEndOfFrame();IEnumerator CaptureScreenshot2(Rect rect){yield return frameEnd;Texture2D screenShot = new Texture2D((int)rect.width, (int)rect.height, TextureFormat.RGB24, false);//先创建一个的空纹理，大小可根据实现需要来设置
#pragma warning disable UNT0017 // SetPixels invocation is slowscreenShot.ReadPixels(rect, 0, 0);//读取屏幕像素信息并存储为纹理数据，
#pragma warning restore UNT0017 // SetPixels invocation is slowscreenShot.Apply();byte[] bytes = screenShot.EncodeToJPG();//然后将这些纹理数据，成一个png图片文件string filename = "e:/qianbo1.JPG";System.IO.File.WriteAllBytes(filename, bytes);Debug.Log(string.Format("截屏了一张图片: {0}", filename));//最后，我返回这个Texture2d对象，这样我们直接，所这个截图图示在游戏中，当然这个根据自己的需求的。//yield return screenShot;}

update

我们使用update 来测试我们的函数，按下空格，按下S，按下F，分别调用函数

 void Update(){if (Input.GetKeyDown(KeyCode.Space)){int screenWidth = Screen.width;int screenHeight = Screen.height;RenderTexture rt = new RenderTexture(screenWidth, screenHeight, 24);Camera.main.targetTexture = rt;Camera.main.Render();RenderTexture.active = rt;//激活这个rt, 并从中中读取像素。Texture2D screenShot = new Texture2D(screenWidth, screenHeight, TextureFormat.RGB24, false);screenShot.ReadPixels(new Rect(0, 0, screenWidth, screenHeight), 0, 0);screenShot.Apply();byte[] bytes = screenShot.EncodeToJPG();//string path = Application.dataPath + "/../" + fileName + ".jpg";//string path = Application.dataPath + "/../" + fileName + ".jpg";string path = "e:/qianbo3.jpg" ;File.WriteAllBytes(path, bytes);Debug.Log(string.Format("space: {0}", path));}else if(Input.GetKeyDown(KeyCode.F)){Rect rect = new Rect(10, 10, 300, 200);CaptureScreenshot2(rect);//Debug.Log(string.Format("F: {0}", path));}else if (Input.GetKeyDown(KeyCode.S)){mainCamera = GameObject.Find("Main Camera");Rect rect = new Rect(0, 0, 300, 200);//获取主摄像机的Camera组件//mainCamera= GameObject.Find("MainCamera").GetComponent<Camera>();//Camera ca = Camera.main;CaptureCamera(Camera.main, rect);}}

以下为存储到磁盘的图像

rtsp 方式传输

我们将使用rtsp方式传送抓取到的纹理图像，使用mjpeg over rtsp， 将jpeg 传送到server，由server提供rtsp 服务，或者http服务，使用c++来写这个server，这个由下一次再讲了。