MR_AR类|ARCore背景渲染 Unity|Shader基础|ar|unity|游戏引擎

文章目录

1.前言
2.ARCore流程
3.渲染流程
- 3.1 数据更新
- 3.2 渲染
4.结语

1.前言像Vuforia、ARCore、EasyAR等sdk，使用时都会将背景与虚拟进行叠加。此功能的实现有很多种，比如直接在Camera的远平面附近添加一个UI或者quad，将camera的画面付给它；比如用CommandBuffer将画面最先渲染；或者重写shader，更改渲染顺序或者设置深度值。ARCore采用的CommandBuffer的方式，但是里面有很多细节需要说明。
2.ARCore流程 ARCore背景渲染脚本为ARCoreBackgroundRenderer，其基本流程为：
1）OnEnable时初始化CommandBuffer
2）Update中实时更新材质的属性，比如相机背景的Texture
3）OnDisable时移除CommandBuffer。
第一步和第三步比较简单，如下代码所示：

private void EnableARBackgroundRendering() { if (BackgroundMaterial == null || m_Camera == null) { return; }m_CameraClearFlags = m_Camera.clearFlags; m_Camera.clearFlags = CameraClearFlags.Depth; m_CommandBuffer = new CommandBuffer(); #if UNITY_ANDROID if (SystemInfo.graphicsMultiThreaded && !InstantPreviewManager.IsProvidingPlatform) { m_CommandBuffer.IssuePluginEvent(ExternApi.ARCoreRenderingUtils_GetRenderEventFunc(), (int)ApiRenderEvent.WaitOnPostUpdateFence); #if UNITY_2018_2_OR_NEWER // There is a bug in Unity that IssuePluginEvent will reset the opengl state but it // doesn't respect the value set to GL.invertCulling. Hence we need to reapply // the invert culling in the command buffer for front camera session. // Note that the CommandBuffer.SetInvertCulling is only available for 2018.2+. var sessionComponent = LifecycleManager.Instance.SessionComponent; if (sessionComponent != null && sessionComponent.DeviceCameraDirection == DeviceCameraDirection.FrontFacing) { m_CommandBuffer.SetInvertCulling(true); } #endif }#endif m_CommandBuffer.Blit(null, BuiltinRenderTextureType.CameraTarget, BackgroundMaterial); m_Camera.AddCommandBuffer(CameraEvent.BeforeForwardOpaque, m_CommandBuffer); m_Camera.AddCommandBuffer(CameraEvent.BeforeGBuffer, m_CommandBuffer); }

private void DisableARBackgroundRendering() { if (m_CommandBuffer == null || m_Camera == null) { return; }m_Camera.clearFlags = m_CameraClearFlags; m_Camera.RemoveCommandBuffer(CameraEvent.BeforeForwardOpaque, m_CommandBuffer); m_Camera.RemoveCommandBuffer(CameraEvent.BeforeGBuffer, m_CommandBuffer); }

3.渲染流程渲染流程从如下两个方面讲解，第一个为数据层，即update方法中实时更新的参数；第二个为渲染层，即CommandBuffer使用材质（shader）。
3.1 数据更新背景渲染时每帧的数据更新如下流程：
1）设置_Brightness，通过此变量实现背景出现时由默认ICON切换到Camera背景的一个小动画。
2）设置_TransitionIconTexTransform，此变量根据默认icon和屏幕大小计算出。如果默认Icon长宽超出屏幕，保证正确裁剪，根据注释以及shader，其计算也是先按比例缩放，然后在平移到中心位置。计算如下：

/// /// Textures transform used in background shader to get texture uv coordinates based on /// screen uv. /// The transformation follows these equations: /// textureUv.x = transform[0] * screenUv.x + transform[1], /// textureUv.y = transform[2] * screenUv.y + transform[3]. /// /// The transform. private Vector4 _TextureTransform() { float transitionWidthTransform = (m_TransitionImageTexture.width - Screen.width) / (2.0f * m_TransitionImageTexture.width); float transitionHeightTransform = (m_TransitionImageTexture.height - Screen.height) / (2.0f * m_TransitionImageTexture.height); return new Vector4( (float)Screen.width / m_TransitionImageTexture.width, transitionWidthTransform, (float)Screen.height / m_TransitionImageTexture.height, transitionHeightTransform); }

注释中有说明如何使用这四个参数。
3）传入视频流Texture，即设置_MainTex。此_MainTex是通过Texture2D.CreateExternalTexture生成的android OES类型texture，并非传统的glTexture，所以在采样时使用的是samplerExternalOES。此Texture通过Frame.CameraImage.Texture获取，并非AcquireCameraImageBytes方法获取的ImageBytes，此ImageBytes为YUV格式数据，并且大小只有640*480；
4）传入_UvTopLeftRight和_UvBottomLeftRight值，此两个值跟2）类似，因为uv并非完全匹配。
3.2 渲染渲染所用的shader为ARCore/ARBackground，此shader较为简单，有两个pass。第一个pass采用GLSL编写，为了从OES纹理采样，并基于输入的参数进行uv裁切（包括初始ARIcon以及_MainTex）。第二个pass则进行最简单的uv采样，不过采样时反转了x值

Shader "ARCore/ARBackground" { Properties { _MainTex ("Main Texture", 2D) = "white" {} _UvTopLeftRight ("UV of top corners", Vector) = (0, 1, 1, 1) _UvBottomLeftRight ("UV of bottom corners", Vector) = (0 , 0, 1, 0) }// For GLES3 or GLES2 on device SubShader { Pass { ZWrite Off Cull OffGLSLPROGRAM#pragma only_renderers gles3 gles// #ifdef SHADER_API_GLES3 cannot take effect because // #extension is processed before any Unity defined symbols. // Use "enable" instead of "require" here, so it only gives a // warning but not compile error when the implementation does not // support the extension. #extension GL_OES_EGL_image_external_essl3 : enable #extension GL_OES_EGL_image_external : enableuniform vec4 _UvTopLeftRight; uniform vec4 _UvBottomLeftRight; // Use the same method in UnityCG.cginc to convert from gamma to // linear space in glsl. vec3 GammaToLinearSpace(vec3 color) { return color * (color * (color * 0.305306011 + 0.682171111) + 0.012522878); }#ifdef VERTEXvarying vec2 textureCoord; varying vec2 uvCoord; void main() { vec2 uvTop = mix(_UvTopLeftRight.xy, _UvTopLeftRight.zw, gl_MultiTexCoord0.x); vec2 uvBottom = mix(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, gl_MultiTexCoord0.x); textureCoord = mix(uvTop, uvBottom, gl_MultiTexCoord0.y); uvCoord = vec2(gl_MultiTexCoord0.x, gl_MultiTexCoord0.y); gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex; }#endif#ifdef FRAGMENT varying vec2 textureCoord; varying vec2 uvCoord; uniform samplerExternalOES _MainTex; uniform sampler2D _TransitionIconTex; uniform vec4 _TransitionIconTexTransform; uniform float _Brightness; void main() { vec3 mainTexColor; #ifdef SHADER_API_GLES3 mainTexColor = texture(_MainTex, textureCoord).rgb; #else mainTexColor = textureExternal(_MainTex, textureCoord).rgb; #endifif (_Brightness < 1.0) { mainTexColor = mainTexColor * _Brightness; if (_TransitionIconTexTransform.x > 0.0 && _TransitionIconTexTransform.z > 0.0) { vec2 uvCoordTex = vec2(uvCoord.x * _TransitionIconTexTransform.x + _TransitionIconTexTransform.y, uvCoord.y * _TransitionIconTexTransform.z + _TransitionIconTexTransform.w); vec4 transitionColor = vec4(0.0); if (uvCoordTex.x >= 0.0 && uvCoordTex.x <= 1.0 && uvCoordTex.y >= 0.0 && uvCoordTex.y <= 1.0) { transitionColor = texture2D(_TransitionIconTex, uvCoordTex); }if (transitionColor.a > 0.0) { mainTexColor = mix(transitionColor.rgb, mainTexColor, _Brightness); } } }#ifndef UNITY_COLORSPACE_GAMMAmainTexColor = GammaToLinearSpace(mainTexColor); #endif gl_FragColor = vec4(mainTexColor, 1.0); }#endifENDGLSL } }// For Instant Preview Subshader { Pass { ZWrite OffCGPROGRAM#pragma exclude_renderers gles3 gles #pragma vertex vert #pragma fragment frag#include "UnityCG.cginc"uniform float4 _UvTopLeftRight; uniform float4 _UvBottomLeftRight; struct appdata { float4 vertex : POSITION; float2 uv : TEXCOORD0; }; struct v2f { float2 uv : TEXCOORD0; float4 vertex : SV_POSITION; }; v2f vert(appdata v) { float2 uvTop = lerp(_UvTopLeftRight.xy, _UvTopLeftRight.zw, v.uv.x); float2 uvBottom = lerp(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, v.uv.x); v2f o; o.vertex = UnityObjectToClipPos(v.vertex); o.uv = lerp(uvTop, uvBottom, v.uv.y); // Instant preview's texture is transformed differently. o.uv = o.uv.yx; o.uv.x = 1.0 - o.uv.x; return o; }sampler2D _MainTex; fixed4 frag(v2f i) : SV_Target { return tex2D(_MainTex, i.uv); }ENDCG } }FallBack Off }

4.结语 【MR_AR类|ARCore背景渲染】如上是ARCore渲染背景的流程，背景渲染最主要一点是OES纹理渲染，当然也可以现在内部转成普通纹理。