Well I am trying to learn more about this and messing around with the Bitsquid engine that doesn't play well with the depth buffer. Oddly enough, on the main menu, before entering the game, the depth buffer is there and works. What I noticed though, is that SMAA did not appear to be seeing the whole depth buffer. I had a random thought, what does it look like when I put "Display Depth" before SMAA, turn it on, and use Color or Luma... turns out it looks great.
The difference, I think, is that Display Depth uses "GetLinearizedDepth" in Reshade.fxh, where as SMAA is using the depth buffer straight. I thought, maybe I can do a copy-pasty kind of deal from Display Depth, but no, it's not like that at all (
at allll). So now need to take Reshade::GetLinearizedDepth into a texture. Either I am doing it the very hard way, or it's kind of hard to do (my coding knowledge is poor at best, and there seems to be several different languages used like c++ vs Reshade vs DXD11?). I'm trying to make a Pixel Shader, basically what was used in Display Depth, write that to a texture on first pass, then take that texture->sampler and substitute it for the normal colorGammaSampler on Luma when I choose the new option in EdgeDetectionType.
I would greatly appreciate some info about where I'm going wrong. I would like to use the method I am currently working on for the sake of learning how to do it, but I would also like to know the way I
should be doing it. Search for LinearDepth, and I think that should get all the sections I worked on. I think a lot of it is ok, but I gave up on the turn the data into a texture part (LinearDepthPS). Not enough info in REFERENCE.md and reverse engineering other shaders with multiple languages at once, or whatever is going on, is not going well.
Thanks,
Vernon.
/**
* _______ ___ ___ ___ ___
* / || \/ | / \ / \
* | (---- | \ / | / ^ \ / ^ \
* \ \ | |\/| | / /_\ \ / /_\ \
* ----) | | | | | / _____ \ / _____ \
* |_______/ |__| |__| /__/ \__\ /__/ \__\
*
* E N H A N C E D
* S U B P I X E L M O R P H O L O G I C A L A N T I A L I A S I N G
*
* for ReShade 3.0
*/
//------------------- Preprocessor Settings -------------------
#ifndef SMAA_PREDICATION
#define SMAA_PREDICATION 0 //[0 or 1] Enables predication which uses BOTH the color and the depth texture for edge detection to more accurately detect edges.
#endif
//----------------------- UI Variables ------------------------
uniform int EdgeDetectionType <
ui_type = "combo";
ui_items = "Luminance edge detection\0Color edge detection\0Depth edge detection\0Luminance of Linearized Depth\0";
ui_label = "Edge Detection Type";
> = 1;
uniform float EdgeDetectionThreshold <
ui_type = "drag";
ui_min = 0.01; ui_max = 0.20; ui_step = 0.005;
ui_label = "Edge Detection Threshold (Luma/Color Detect)";
ui_tooltip = "Edge detection threshold. If SMAA misses some edges try lowering this slightly.";
> = 0.10;
uniform float DepthDetectThreshold <
ui_type = "drag";
ui_min = 0.005; ui_max = 0.02; ui_step = 0.0005;
ui_label = "Depth Detection Threshold";
ui_tooltip = "Edge detection threshold when using Depth Detection.";
> = 0.01;
uniform int MaxSearchSteps <
ui_type = "drag";
ui_min = 0; ui_max = 256;
ui_label = "Max Search Steps";
ui_tooltip = "Determines the radius SMAA will search for aliased edges.";
> = 98;
uniform int MaxSearchStepsDiagonal <
ui_type = "drag";
ui_min = 0; ui_max = 64;
ui_label = "Max Search Steps Diagonal";
ui_tooltip = "Determines the radius SMAA will search for diagonal aliased edges";
> = 16;
uniform int CornerRounding <
ui_type = "drag";
ui_min = 0; ui_max = 100;
ui_label = "Corner Rounding";
ui_tooltip = "Determines the percent of anti-aliasing to apply to corners.";
> = 0;
uniform float LocalContrastAdapt <
ui_type = "drag";
ui_min = 0; ui_max = 10; ui_step = 0.1;
ui_label = "Local Contrast Adapt";
ui_tooltip = "Factor of contrast between two neighboring edges to discard one if dominated by other (smaller = discard sooner)";
> = 2.0;
#if SMAA_PREDICATION
uniform float PredicationThreshold <
ui_type = "drag";
ui_min = 0.005; ui_max = 1.00; ui_step = 0.01;
ui_tooltip = "Threshold to be used in the additional predication buffer.";
ui_label = "Predication Threshold";
> = 0.01;
uniform float PredicationScale <
ui_type = "drag";
ui_min = 0; ui_max = 8;
ui_tooltip = "How much to scale the global threshold used for luma or color edge.";
ui_label = "Predication Scale";
> = 0.2;
uniform float PredicationStrength <
ui_type = "drag";
ui_min = 0; ui_max = 4;
ui_tooltip = "How much to locally decrease the threshold.";
ui_label = "Predication Strength";
> = 0.4;
#endif
uniform int DebugOutput <
ui_type = "combo";
ui_items = "None\0'edgesTex' buffer\0'blendTex' buffer\0'LinearDepthTex' buffer\0";
ui_label = "Debug Output";
> = false;
#define SMAA_RT_METRICS float4(BUFFER_RCP_WIDTH, BUFFER_RCP_HEIGHT, BUFFER_WIDTH, BUFFER_HEIGHT)
#define SMAA_CUSTOM_SL 1
#define SMAA_PRESET_CUSTOM 1
#define SMAA_THRESHOLD EdgeDetectionThreshold
#define SMAA_DEPTH_THRESHOLD DepthDetectThreshold
#define SMAA_MAX_SEARCH_STEPS MaxSearchSteps
#define SMAA_MAX_SEARCH_STEPS_DIAG MaxSearchStepsDiagonal
#define SMAA_CORNER_ROUNDING CornerRounding
#define SMAA_LOCAL_CONTRAST_ADAPTATION_FACTOR LocalContrastAdapt
#if SMAA_PREDICATION
#define SMAA_PREDICATION_THRESHOLD PredicationThreshold
#define SMAA_PREDICATION_SCALE PredicationScale
#define SMAA_PREDICATION_STRENGTH PredicationStrength
#endif
#define predicationSampler ReShade::DepthBuffer
#define SMAATexture2D(tex) sampler tex
#define SMAATexturePass2D(tex) tex
#define SMAASampleLevelZero(tex, coord) tex2Dlod(tex, float4(coord, coord))
#define SMAASampleLevelZeroPoint(tex, coord) SMAASampleLevelZero(tex, coord)
#define SMAASampleLevelZeroOffset(tex, coord, offset) tex2Dlodoffset(tex, float4(coord, coord), offset)
#define SMAASample(tex, coord) tex2D(tex, coord)
#define SMAASamplePoint(tex, coord) SMAASample(tex, coord)
#define SMAASampleOffset(tex, coord, offset) tex2Doffset(tex, coord, offset)
#define SMAA_BRANCH [branch]
#define SMAA_FLATTEN [flatten]
#if (__RENDERER__ == 0xb000 || __RENDERER__ == 0xb100)
#define SMAAGather(tex, coord) tex2Dgather(tex, coord, 0)
#endif
#include "SMAA.fxh"
#include "ReShade.fxh"
// Textures
texture LinearDepthTex
{
Width = BUFFER_WIDTH;
Height = BUFFER_HEIGHT;
Format = RGBA8;
};
texture edgesTex
{
Width = BUFFER_WIDTH;
Height = BUFFER_HEIGHT;
Format = RG8;
};
texture blendTex
{
Width = BUFFER_WIDTH;
Height = BUFFER_HEIGHT;
Format = RGBA8;
};
texture areaTex < source = "AreaTex.dds"; >
{
Width = 160;
Height = 560;
Format = RG8;
};
texture searchTex < source = "SearchTex.dds"; >
{
Width = 64;
Height = 16;
Format = R8;
};
// Samplers
sampler LinearDepthSampler
{
Texture = LinearDepthTex;
AddressU = Clamp; AddressV = Clamp;
MipFilter = Linear; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = false;
};
sampler colorGammaSampler
{
Texture = ReShade::BackBufferTex;
AddressU = Clamp; AddressV = Clamp;
MipFilter = Point; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = false;
};
sampler colorLinearSampler
{
Texture = ReShade::BackBufferTex;
AddressU = Clamp; AddressV = Clamp;
MipFilter = Point; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = true;
};
sampler edgesSampler
{
Texture = edgesTex;
AddressU = Clamp; AddressV = Clamp;
MipFilter = Linear; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = false;
};
sampler blendSampler
{
Texture = blendTex;
AddressU = Clamp; AddressV = Clamp;
MipFilter = Linear; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = false;
};
sampler areaSampler
{
Texture = areaTex;
AddressU = Clamp; AddressV = Clamp; AddressW = Clamp;
MipFilter = Linear; MinFilter = Linear; MagFilter = Linear;
SRGBTexture = false;
};
sampler searchSampler
{
Texture = searchTex;
AddressU = Clamp; AddressV = Clamp; AddressW = Clamp;
MipFilter = Point; MinFilter = Point; MagFilter = Point;
SRGBTexture = false;
};
// Vertex shaders
void SMAAEdgeDetectionWrapVS(
in uint id : SV_VertexID,
out float4 position : SV_Position,
out float2 texcoord : TEXCOORD0,
out float4 offset[3] : TEXCOORD1)
{
PostProcessVS(id, position, texcoord);
SMAAEdgeDetectionVS(texcoord, offset);
}
void SMAABlendingWeightCalculationWrapVS(
in uint id : SV_VertexID,
out float4 position : SV_Position,
out float2 texcoord : TEXCOORD0,
out float2 pixcoord : TEXCOORD1,
out float4 offset[3] : TEXCOORD2)
{
PostProcessVS(id, position, texcoord);
SMAABlendingWeightCalculationVS(texcoord, pixcoord, offset);
}
void SMAANeighborhoodBlendingWrapVS(
in uint id : SV_VertexID,
out float4 position : SV_Position,
out float2 texcoord : TEXCOORD0,
out float4 offset : TEXCOORD1)
{
PostProcessVS(id, position, texcoord);
SMAANeighborhoodBlendingVS(texcoord, offset);
}
// Pixel shaders
void LinearDepthPS(
in float4 position : SV_Position,
in float2 texcoord : TEXCOORD0,
out float4 LiniOut)
{
//was using just R8 texture, but I think texture into SMAA.fxh needs to be RGBA8, so I stuck 1.0 on the end for alpha and then it screamed at me about Stencil Buffer.
LiniOut = (ReShade::GetLinearizedDepth(texcoord).rrr,1.0);
}
float2 SMAAEdgeDetectionWrapPS(
float4 position : SV_Position,
float2 texcoord : TEXCOORD0,
float4 offset[3] : TEXCOORD1) : SV_Target
{
if (EdgeDetectionType == 0)
return SMAALumaEdgeDetectionPS(texcoord, offset, colorGammaSampler
#if SMAA_PREDICATION
,predicationSampler
#endif
);
if (EdgeDetectionType == 1)
return SMAAColorEdgeDetectionPS(texcoord, offset, colorGammaSampler
#if SMAA_PREDICATION
,predicationSampler
#endif
);
if (EdgeDetectionType == 2)
return SMAADepthEdgeDetectionPS(texcoord, offset, ReShade::DepthBuffer);
return SMAALumaEdgeDetectionPS(texcoord, offset, LinearDepthSampler
#if SMAA_PREDICATION
,predicationSampler
#endif
);
}
float4 SMAABlendingWeightCalculationWrapPS(
float4 position : SV_Position,
float2 texcoord : TEXCOORD0,
float2 pixcoord : TEXCOORD1,
float4 offset[3] : TEXCOORD2) : SV_Target
{
return SMAABlendingWeightCalculationPS(texcoord, pixcoord, offset, edgesSampler, areaSampler, searchSampler, 0.0);
}
float3 SMAANeighborhoodBlendingWrapPS(
float4 position : SV_Position,
float2 texcoord : TEXCOORD0,
float4 offset : TEXCOORD1) : SV_Target
{
if (DebugOutput == 1)
return tex2D(edgesSampler, texcoord).rgb;
if (DebugOutput == 2)
return tex2D(blendSampler, texcoord).rgb;
if (DebugOutput == 3)
return tex2D(LinearDepthSampler, texcoord).rgb;
return SMAANeighborhoodBlendingPS(texcoord, offset, colorLinearSampler, blendSampler).rgb;
}
// Rendering passes
technique SMAA
{
pass LinearDepthPass
{
#if (EdgeDetectionType == 3)
VertexShader = Reshade::PostProcessVS;
PixelShader = LinearDepthPS;
RenderTarget = LinearDepthTex;
ClearRenderTargets = true;
#endif
}
pass EdgeDetectionPass
{
VertexShader = SMAAEdgeDetectionWrapVS;
PixelShader = SMAAEdgeDetectionWrapPS;
RenderTarget = edgesTex;
ClearRenderTargets = true;
StencilEnable = true;
StencilPass = REPLACE;
StencilRef = 1;
}
pass BlendWeightCalculationPass
{
VertexShader = SMAABlendingWeightCalculationWrapVS;
PixelShader = SMAABlendingWeightCalculationWrapPS;
RenderTarget = blendTex;
ClearRenderTargets = true;
StencilEnable = true;
StencilPass = KEEP;
StencilFunc = EQUAL;
StencilRef = 1;
}
pass NeighborhoodBlendingPass
{
VertexShader = SMAANeighborhoodBlendingWrapVS;
PixelShader = SMAANeighborhoodBlendingWrapPS;
StencilEnable = false;
SRGBWriteEnable = true;
}
}
