前言

前端hololens2(unity c#)，后端linux gpu服务器(python)

目标：

前端上传图片给后端，后端发送json至前端。

解释：

由于我们设定的帧率为25fps，因此每0.04秒，前端的hololens2将获取一张图片发送给后端。后端将预测得到的结果发送给前端。由于前端发送照片的速度和后端向前端发送预测结果的频率都取决于帧率，因此可以将上传与下传的步骤合二为一，在一次请求中完成。

这可以使用到http请求（http request）的post方式。http的请求方式包括get、put、post、patch等，最常用的就是get和post，get一般用于获取信息，post用于上传信息并获取信息。http请求简单来说就是，后端提供服务，前端向后端发送一个请求（request），后端就会给前端一个响应（response）。

后端

python做后端服务可以使用很多现成的方案，我就随便跳了一个，叫fastapi，这是官网文档：https://fastapi.tiangolo.com/zh/

环境准备：按照官网的指示，安装以下三个包

代码：

test.py

# -*- coding:utf-8 -*-
from typing import Union
 
from fastapi import FastAPI, File, UploadFile
from pydantic import BaseModel
 
import uvicorn
 
app = FastAPI()
 
 
@app.post("/uploadfile/")
async def create_upload_file(file: UploadFile):
    return {"filename": file.filename}  # 返回一个json
 
 
if __name__ == '__main__':
    uvicorn.run(app="test:app", host="0.0.0.0", port=8000, reload=True)
# "test:app"中的test是文件名（test.py）。也是相对路径，如果启动命令不是当前文件则改为相对路径。
# host参数如果设为107.0.0.1则只为当前设备提供服务，如果要为任意设备提供服务则填0.0.0.0
# 端口任意，和前端一致即可。注意可能需要联系管理员开放这个端口。

前端

关于untiy脚本如何获取图片，官方文档Unity 中的照片/视频摄像头 - Mixed Reality | Microsoft Learn

上传到后端的部分参考了如下代码https://blog.csdn.net/Apple_Coco/article/details/113840722

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Windows.WebCam;
using System.Linq;
using UnityEngine.Networking;
 
 
public class test : MonoBehaviour
{
 
 
    // Start is called before the first frame update
    void Start()
    {
 
    }
 
    // Update is called once per frame
    void Update()
    {
 
    }
 
    // FixUpdate is called once per phsic frame
    private void FixedUpdate()
    {
        MyPhotoCapture myPhotoCapture = new MyPhotoCapture();
        myPhotoCapture.StartCapture();
    }
}
 
 
public class MyPhotoCapture : MonoBehaviour
{
    PhotoCapture photoCaptureObject = null;
 
    internal bool captureIsActive;
 
    public void StartCapture()
    {
        if (!captureIsActive)
        {
            captureIsActive = true;
            PhotoCapture.CreateAsync(false, OnPhotoCaptureCreated);
        }
        else
        {
            captureIsActive = false;
        }
    }
 
    void OnPhotoCaptureCreated(PhotoCapture captureObject)
    {
        photoCaptureObject = captureObject;
 
        Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
 
 
        var cameraParams = new CameraParameters()
        {
            hologramOpacity = 0f,
            cameraResolutionWidth = cameraResolution.width,
            cameraResolutionHeight = cameraResolution.height,
            pixelFormat = CapturePixelFormat.JPEG
        };
 
        captureObject.StartPhotoModeAsync(cameraParams, OnPhotoModeStarted);
 
    }
 
    private void OnPhotoModeStarted(PhotoCapture.PhotoCaptureResult result)
    {
        if (result.success)
        {
            //string filename = string.Format(@"CapturedImage{0}_n.jpg", Time.time);
            //string filePath = System.IO.Path.Combine(Application.persistentDataPath, filename);
            //// 保存到磁盘
            //photoCaptureObject.TakePhotoAsync(filePath, PhotoCaptureFileOutputFormat.JPG, OnCapturedPhotoToDisk);
            //// 保存到内存
            //photoCaptureObject.TakePhotoAsync(OnCapturedPhotoToMemory);
            // 上传到服务器
            photoCaptureObject.TakePhotoAsync((photoCaptureResult, frame) =>
            {
                if (photoCaptureResult.success)
                {
                    Debug.Log("Photo capture done.");
 
                    var buffer = new List<byte>();
                    frame.CopyRawImageDataIntoBuffer(buffer);
                    StartCoroutine(CustomVisionAnalyser.Instance.AnalyseLastImageCaptured(buffer.ToArray()));
                }
                photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
            });
        }
        else
        {
            Debug.LogError("Unable to start photo mode!");
        }
    }
 
    //void OnCapturedPhotoToDisk(PhotoCapture.PhotoCaptureResult result)
    //{
    //    if (result.success)
    //    {
    //        Debug.Log("Saved Photo to disk!");
    //        photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
    //    }
    //    else
    //    {
    //        Debug.Log("Failed to save Photo to disk");
    //    }
    //}
 
    //void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
    //{
    //    if (result.success)
    //    {
    //        // Create our Texture2D for use and set the correct resolution
    //        Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
    //        Texture2D targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);
    //        // Copy the raw image data into our target texture
    //        photoCaptureFrame.UploadImageDataToTexture(targetTexture);
    //        // Do as we wish with the texture such as apply it to a material, etc.
    //    }
    //    // Clean up
    //    photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
    //}
 
    void OnStoppedPhotoMode(PhotoCapture.PhotoCaptureResult result)
    {
        photoCaptureObject.Dispose();
        photoCaptureObject = null;
 
        captureIsActive = false;
    }
}
 
public class CustomVisionAnalyser : MonoBehaviour
{
 
    public static CustomVisionAnalyser Instance;
    private string predictionEndpoint = "http://123.56.44.128:8000/uploadfile";
 
    private void Awake()
    {
        Instance = this;
    }
 
    /// <summary>
    /// Call the Computer Vision Service to submit the image.
    /// </summary>
    public IEnumerator AnalyseLastImageCaptured(byte[] imageBytes)
    {
        WWWForm webForm = new WWWForm();
 
        // 将图片byte数组添加进表单
        webForm.AddBinaryData("file", imageBytes, "photo.jpg");
 
        // 将图片上传到服务器
        using (UnityWebRequest unityWebRequest = UnityWebRequest.Post(predictionEndpoint, webForm))
        {
            // The download handler will help receiving the analysis from Azure
            unityWebRequest.downloadHandler = new DownloadHandlerBuffer();
 
            // Send the request
            yield return unityWebRequest.SendWebRequest();
 
            if (unityWebRequest.isHttpError || unityWebRequest.isNetworkError)
            {
                Debug.Log(unityWebRequest.error);
            }
            else
            {
                string response = unityWebRequest.downloadHandler.text;
                Debug.Log(response);
            }
        }
    }
}

这里你只要把这个脚本挂到任意一个游戏对象里就可以执行。只要简单了解一下unity生命周期，就能知道FixUpdate那个函数为什么可以每格固定时间就调用一次。