Merge branch 'tangentspace' into 'master'

Untangle tangent space handling and parallax

See merge request OpenMW/openmw!3648

components/shader/shadervisitor.cpp

@@ -304,14 +304,6 @@ namespace Shader
         if (node.getUserValue(Misc::OsgUserValues::sXSoftEffect, softEffect) && softEffect)
             mRequirements.back().mSoftParticles = true;
 
-        int applyMode;
-        // Oblivion parallax
-        if (node.getUserValue("applyMode", applyMode) && applyMode == 4)
-        {
-            mRequirements.back().mShaderRequired = true;
-            mRequirements.back().mDiffuseHeight = true;
-        }
-
         // Make sure to disregard any state that came from a previous call to createProgram
         osg::ref_ptr<AddedState> addedState = getAddedState(*stateset);
 
@@ -359,7 +351,17 @@ namespace Shader
                                 normalMap = texture;
                             }
                             else if (texName == "diffuseMap")
+                            {
+                                int applyMode;
+                                // Oblivion parallax
+                                if (node.getUserValue("applyMode", applyMode) && applyMode == 4)
+                                {
+                                    mRequirements.back().mShaderRequired = true;
+                                    mRequirements.back().mDiffuseHeight = true;
+                                    mRequirements.back().mTexStageRequiringTangents = unit;
+                                }
                                 diffuseMap = texture;
+                            }
                             else if (texName == "specularMap")
                                 specularMap = texture;
                             else if (texName == "bumpMap")

files/shaders/CMakeLists.txt

@@ -45,6 +45,7 @@ set(SHADER_FILES
     compatibility/shadowcasting.vert
     compatibility/shadowcasting.frag
     compatibility/vertexcolors.glsl
+    compatibility/normals.glsl
     compatibility/multiview_resolve.vert
     compatibility/multiview_resolve.frag
     compatibility/depthclipped.vert

files/shaders/compatibility/bs/default.frag

@@ -24,7 +24,6 @@ varying vec2 emissiveMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 varying float euclideanDepth;
@@ -46,11 +45,10 @@ uniform bool useTreeAnim;
 #include "compatibility/vertexcolors.glsl"
 #include "compatibility/shadows_fragment.glsl"
 #include "compatibility/fog.glsl"
+#include "compatibility/normals.glsl"
 
 void main()
 {
-    vec3 normal = normalize(passNormal);
-
 #if @diffuseMap
     gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV);
     gl_FragData[0].a *= coveragePreservingAlphaScale(diffuseMap, diffuseMapUV);
@@ -65,15 +63,10 @@ void main()
 
 #if @normalMap
     vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
+    vec3 viewNormal = normalToView(normalTex.xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
 #endif
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
 
     float shadowing = unshadowedLightRatio(linearDepth);
     vec3 diffuseLight, ambientLight;

files/shaders/compatibility/bs/default.vert

@@ -36,6 +36,7 @@ varying vec3 passNormal;
 
 #include "compatibility/vertexcolors.glsl"
 #include "compatibility/shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 
 void main(void)
 {
@@ -45,6 +46,14 @@ void main(void)
     gl_ClipVertex = viewPos;
     euclideanDepth = length(viewPos.xyz);
     linearDepth = getLinearDepth(gl_Position.z, viewPos.z);
+    passColor = gl_Color;
+    passViewPos = viewPos.xyz;
+    passNormal = gl_Normal.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+
+#if @normalMap
+    normalToViewMatrix *= generateTangentSpace(gl_MultiTexCoord7.xyzw, passNormal);
+#endif
 
 #if @diffuseMap
     diffuseMapUV = (gl_TextureMatrix[@diffuseMapUV] * gl_MultiTexCoord@diffuseMapUV).xy;
@@ -56,15 +65,11 @@ void main(void)
 
 #if @normalMap
     normalMapUV = (gl_TextureMatrix[@normalMapUV] * gl_MultiTexCoord@normalMapUV).xy;
-    passTangent = gl_MultiTexCoord7.xyzw;
 #endif
 
-    passColor = gl_Color;
-    passViewPos = viewPos.xyz;
-    passNormal = gl_Normal.xyz;
 
 #if @shadows_enabled
-    vec3 viewNormal = normalize((gl_NormalMatrix * gl_Normal).xyz);
+    vec3 viewNormal = normalToView(passNormal);
     setupShadowCoords(viewPos, viewNormal);
 #endif
 }

files/shaders/compatibility/groundcover.frag

@@ -18,7 +18,6 @@ varying vec2 diffuseMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 // Other shaders respect forcePPL, but legacy groundcover mods were designed to work with vertex lighting.
@@ -44,23 +43,10 @@ varying vec3 passNormal;
 #include "lib/light/lighting.glsl"
 #include "lib/material/alpha.glsl"
 #include "fog.glsl"
+#include "compatibility/normals.glsl"
 
 void main()
 {
-    vec3 normal = normalize(passNormal);
-
-#if @normalMap
-    vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
-
 #if @diffuseMap
     gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV);
 #else
@@ -72,6 +58,12 @@ void main()
 
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
+#if @normalMap
+    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV).xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
+#endif
+
     float shadowing = unshadowedLightRatio(linearDepth);
 
     vec3 lighting;

files/shaders/compatibility/groundcover.vert

@@ -21,7 +21,6 @@ varying vec2 diffuseMapUV;
 
 #if @normalMap
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 // Other shaders respect forcePPL, but legacy groundcover mods were designed to work with vertex lighting.
@@ -41,6 +40,7 @@ centroid varying vec3 shadowDiffuseLighting;
 varying vec3 passNormal;
 
 #include "shadows_vertex.glsl"
+#include "compatibility/normals.glsl"
 #include "lib/light/lighting.glsl"
 #include "lib/view/depth.glsl"
 
@@ -149,8 +149,14 @@ void main(void)
 
     linearDepth = getLinearDepth(gl_Position.z, viewPos.z);
 
+    passNormal = rotation3(rotation) * gl_Normal.xyz;
+    normalToViewMatrix = gl_NormalMatrix;
+#if @normalMap
+    normalToViewMatrix *= generateTangentSpace(gl_MultiTexCoord7.xyzw * rotation, passNormal);
+#endif
+
 #if (!PER_PIXEL_LIGHTING || @shadows_enabled)
-    vec3 viewNormal = normalize((gl_NormalMatrix * rotation3(rotation) * gl_Normal).xyz);
+    vec3 viewNormal = normalToView(passNormal);
 #endif
 
 #if @diffuseMap
@@ -159,10 +165,8 @@ void main(void)
 
 #if @normalMap
     normalMapUV = (gl_TextureMatrix[@normalMapUV] * gl_MultiTexCoord@normalMapUV).xy;
-    passTangent = gl_MultiTexCoord7.xyzw * rotation;
 #endif
 
-    passNormal = rotation3(rotation) * gl_Normal.xyz;
 #if PER_PIXEL_LIGHTING
     passViewPos = viewPos.xyz;
 #else

files/shaders/compatibility/normals.glsl

@@ -0,0 +1,14 @@
+varying mat3 normalToViewMatrix;
+
+mat3 generateTangentSpace(vec4 tangent, vec3 normal)
+{
+    vec3 normalizedNormal = normalize(normal);
+    vec3 normalizedTangent = normalize(tangent.xyz);
+    vec3 binormal = cross(normalizedTangent, normalizedNormal) * tangent.w;
+    return mat3(normalizedTangent, binormal, normalizedNormal);
+}
+
+vec3 normalToView(vec3 normal)
+{
+    return normalize(normalToViewMatrix * normal);
+}

files/shaders/compatibility/objects.frag

@@ -37,7 +37,6 @@ varying vec2 emissiveMapUV;
 #if @normalMap
 uniform sampler2D normalMap;
 varying vec2 normalMapUV;
-varying vec4 passTangent;
 #endif
 
 #if @envMap
@@ -79,6 +78,9 @@ uniform float emissiveMult;
 uniform float specStrength;
 varying vec3 passViewPos;
 varying vec3 passNormal;
+#if @normalMap || @diffuseParallax
+varying vec4 passTangent;
+#endif
 
 #if @additiveBlending
 #define ADDITIVE_BLENDING
@@ -91,6 +93,7 @@ varying vec3 passNormal;
 #include "fog.glsl"
 #include "vertexcolors.glsl"
 #include "shadows_fragment.glsl"
+#include "compatibility/normals.glsl"
 
 #if @softParticles
 #include "lib/particle/soft.glsl"
@@ -113,62 +116,32 @@ void main()
     applyOcclusionDiscard(orthoDepthMapCoord, texture2D(orthoDepthMap, orthoDepthMapCoord.xy * 0.5 + 0.5).r);
 #endif
 
-    vec3 normal = normalize(passNormal);
-    vec3 viewVec = normalize(passViewPos.xyz);
+    // only offset diffuse and normal maps for now, other textures are more likely to be using a completely different UV set
+    vec2 offset = vec2(0.0);
 
-#if @normalMap
-    vec4 normalTex = texture2D(normalMap, normalMapUV);
-
-    vec3 normalizedNormal = normal;
-    vec3 normalizedTangent = normalize(passTangent.xyz);
-    vec3 binormal = cross(normalizedTangent, normalizedNormal) * passTangent.w;
-    mat3 tbnTranspose = mat3(normalizedTangent, binormal, normalizedNormal);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-
-#if !@diffuseMap
-    gl_FragData[0] = vec4(1.0);
-#else
-    vec2 adjustedDiffuseUV = diffuseMapUV;
-
-#if @normalMap && (@parallax || @diffuseParallax)
-    vec3 cameraPos = (gl_ModelViewMatrixInverse * vec4(0,0,0,1)).xyz;
-    vec3 objectPos = (gl_ModelViewMatrixInverse * vec4(passViewPos, 1)).xyz;
-    vec3 eyeDir = normalize(cameraPos - objectPos);
+#if @parallax || @diffuseParallax
 #if @parallax
-    float height = normalTex.a;
+    float height = texture2D(normalMap, normalMapUV).a;
     float flipY = (passTangent.w > 0.0) ? -1.f : 1.f;
 #else
     float height = texture2D(diffuseMap, diffuseMapUV).a;
     // FIXME: shouldn't be necessary, but in this path false-positives are common
     float flipY = -1.f;
 #endif
-    vec2 offset = getParallaxOffset(eyeDir, tbnTranspose, height, flipY);
-    adjustedDiffuseUV += offset; // only offset diffuse for now, other textures are more likely to be using a completely different UV set
-
-    // TODO: check not working as the same UV buffer is being bound to different targets
-    // if diffuseMapUV == normalMapUV
-#if 1
-    // fetch a new normal using updated coordinates
-    normalTex = texture2D(normalMap, adjustedDiffuseUV);
-
-    normal = normalize(tbnTranspose * (normalTex.xyz * 2.0 - 1.0));
-#endif
-
+    offset = getParallaxOffset(transpose(normalToViewMatrix) * normalize(-passViewPos), height, flipY);
 #endif
 
-    vec4 diffuseTex = texture2D(diffuseMap, adjustedDiffuseUV);
-    gl_FragData[0].xyz = diffuseTex.xyz;
-#if !@diffuseParallax
-    gl_FragData[0].a = diffuseTex.a * coveragePreservingAlphaScale(diffuseMap, adjustedDiffuseUV);
-#else
+#if @diffuseMap
+    gl_FragData[0] = texture2D(diffuseMap, diffuseMapUV + offset);
+#if @diffuseParallax
     gl_FragData[0].a = 1.0;
+#else
+    gl_FragData[0].a *= coveragePreservingAlphaScale(diffuseMap, diffuseMapUV + offset);
 #endif
+#else
+    gl_FragData[0] = vec4(1.0);
 #endif
 
-    vec3 viewNormal = normalize(gl_NormalMatrix * normal);
-
     vec4 diffuseColor = getDiffuseColor();
     gl_FragData[0].a *= diffuseColor.a;
 
@@ -179,6 +152,14 @@ void main()
 
     gl_FragData[0].a = alphaTest(gl_FragData[0].a, alphaRef);
 
+#if @normalMap
+    vec3 viewNormal = normalToView(texture2D(normalMap, normalMapUV + offset).xyz * 2.0 - 1.0);
+#else
+    vec3 viewNormal = normalToView(normalize(passNormal));
+#endif
+
+    vec3 viewVec = normalize(passViewPos);
+
 #if @detailMap
     gl_FragData[0].xyz *= texture2D(detailMap, detailMapUV).xyz * 2.0;
 #endif