Welcome, Guest.
Username: Password: Remember me

TOPIC: Help needed to get a working depth check to Emboss

Help needed to get a working depth check to Emboss 1 year 6 months ago #1

Hey!

I'm trying to add a working depth check to the Emboss shader but I have no idea how to approach it honestly.
At the moment, something does seem to happen when turning the depth cutoff knob, but far from what I want.

This is the current code:
/*
	Full credits to the ReShade team
	Ported by Insomnia
*/

uniform float fEmbossPower <
	ui_type = "drag";
	ui_min = 0.01; ui_max = 2.0;
	ui_label = "Emboss Power";
> = 0.150;
uniform float fEmbossOffset <
	ui_type = "drag";
	ui_min = 0.1; ui_max = 5.0;
	ui_label = "Emboss Offset";
> = 1.00;
uniform float iEmbossAngle <
	ui_type = "drag";
	ui_min = 0.0; ui_max = 360.0;
	ui_label = "Emboss Angle";
> = 90.00;





//#define bEmbossDoDepthCheck 1 //-EXPERIMENTAL! If enabled, shader compares emboss samples depth to avoid artifacts at object borders.
//#define fEmbossDepthCutoff 0.0001 //-Preserves object edges from getting artifacts. If pixel depth difference of emboss samples is higher than that, pixel gets skipped.

uniform float fEmbossDepthCutoff <
	ui_label = "fEmbossDepthCutoff";
	ui_type = "drag";
	ui_min = 0.01; ui_max = 1.0;
	> = 0.2;

#include "ReShade.fxh"

float3 EmbossPass(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target
{
	float3 res = 0;
	float3 color = tex2D(ReShade::BackBuffer, texcoord).rgb;

    float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);
	float3 col1 = tex2D(ReShade::BackBuffer, texcoord - ReShade::PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = color.rgb;
	float3 col3 = tex2D(ReShade::BackBuffer, texcoord + ReShade::PixelSize*fEmbossOffset*offset).rgb;

		float depth1 = tex2D(ReShade::DepthBuffer,texcoord - ReShade::PixelSize*fEmbossOffset).rgb;
		float depth2 = tex2D(ReShade::DepthBuffer,texcoord).rgb;
		float depth3 = tex2D(ReShade::DepthBuffer,texcoord + ReShade::PixelSize*fEmbossOffset).rgb;

	
	float3 colEmboss = col1 * 2.0 - col2 - col3;

	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;

	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );

	color.xyz = lerp( colFinal, col2, luminance * luminance ).xyz;


        if(max(abs(depth1-depth2),abs(depth3-depth2)) > fEmbossDepthCutoff) color = res;


	return color;

}

technique Emboss_Tech
{
	pass Emboss
	{
		VertexShader = PostProcessVS;
		PixelShader = EmbossPass;
	}
}


Thanks for any input! :)
Last Edit: 1 year 6 months ago by Insomnia.
The administrator has disabled public write access.
The following user(s) said Thank You: Aelius Maximus

Help needed to get a working depth check to Emboss 1 year 6 months ago #2

I'm not quite sure what exactly you are trying to do, but the "color = res;" line looks wrong, since "res" is never initialized to anything other than zero (first line in the function). I doubt you want the picture to be black for pixels where your check succeeds, so you probably wanted to set "res" to the original input color:
float3 color = res = tex2D(ReShade::BackBuffer, texcoord).rgb;
Cheers, crosire =)
The administrator has disabled public write access.
The following user(s) said Thank You: Insomnia

Help needed to get a working depth check to Emboss 1 year 6 months ago #3

crosire wrote:
I'm not quite sure what exactly you are trying to do
Neither do I lol.
but the "color = res;" line looks wrong, since "res" is never initialized to anything other than zero (first line in the function). I doubt you want the picture to be black for pixels where your check succeeds, so you probably wanted to set "res" to the original input color:
float3 color = res = tex2D(ReShade::BackBuffer, texcoord).rgb;

Thanks, but not much happens I'm afraid.

Here's the original code:
NAMESPACE_ENTER(CFX)

#include CFX_SETTINGS_DEF

#if USE_EMBOSS

float4 PS_Emboss(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target
{
	float4 res = 0;
	float4 origcolor = tex2D(RFX_backbufferColor, texcoord);

        float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);
	float3 col1 = tex2D(RFX_backbufferColor, texcoord - RFX_PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = origcolor.rgb;
	float3 col3 = tex2D(RFX_backbufferColor, texcoord + RFX_PixelSize*fEmbossOffset*offset).rgb;

#if(bEmbossDoDepthCheck != 0)
	float depth1 = tex2D(RFX_depthColor,texcoord - RFX_PixelSize*fEmbossOffset).r;
	float depth2 = tex2D(RFX_depthColor,texcoord).r;
	float depth3 = tex2D(RFX_depthColor,texcoord + RFX_PixelSize*fEmbossOffset).r;
#endif
	
	float3 colEmboss = col1 * 2.0 - col2 - col3;

	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;

	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );

	res.xyz = lerp( colFinal, col2, luminance * luminance ).xyz;
#if(bEmbossDoDepthCheck != 0)
        if(max(abs(depth1-depth2),abs(depth3-depth2)) > fEmbossDepthCutoff) res = origcolor;
#endif
	return res;

}

technique Emboss_Tech <bool enabled = RFX_Start_Enabled; int toggle = Emboss_ToggleKey; >
{
	pass Emboss
	{
		VertexShader = RFX_VS_PostProcess;
		PixelShader = PS_Emboss;
	}
}

#endif

#include CFX_SETTINGS_UNDEF

NAMESPACE_LEAVE()

I've just been fooling around to try and add a working depth function on my own, but like you've noticed I don't know what the heck I'm doing.
Last Edit: 1 year 6 months ago by Insomnia.
The administrator has disabled public write access.

Help needed to get a working depth check to Emboss 1 year 6 months ago #4

This definitely has potential, hope you get it working!
The administrator has disabled public write access.

Help needed to get a working depth check to Emboss 1 year 5 months ago #5

Been trying to do... something, but I can't get it to work. I'm simply not meant for programming as I don't understand it.

Gonna drop this.
The administrator has disabled public write access.

Help needed to get a working depth check to Emboss 1 year 5 months ago #6

I got you buddy ;)

I didn't touch the algorithm at all, just ported the code to ReShade 3.0. It would be faster with bDepthCheck as a preprocessor setting instead of a uniform, but this way you can enable/disable it in real time.

Good call including the original code. Was easier to use as a starting point.
#include "ReShade.fxh"

uniform float fEmbossPower <
	ui_type = "drag";
	ui_min = 0.01; ui_max = 2.0;
	ui_label = "Emboss Power";
> = 0.150;
uniform float fEmbossOffset <
	ui_type = "drag";
	ui_min = 0.1; ui_max = 5.0;
	ui_label = "Emboss Offset";
> = 1.00;
uniform float iEmbossAngle <
	ui_type = "drag";
	ui_min = 0.0; ui_max = 360.0;
	ui_label = "Emboss Angle";
> = 90.00;

uniform bool bDepthCheck <
	ui_label = "Enable Depth Check";
> = false;

uniform float fDepthCutoff <
	ui_type = "input";
	ui_label = "Depth Cutoff";
> = 0.001;

float4 EmbossPS(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target {
	float3 res;
	float3 origcolor = tex2D(ReShade::BackBuffer, texcoord).rgb;

	float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);

	float3 col1 = tex2D(ReShade::BackBuffer, texcoord - ReShade::PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = origcolor;
	float3 col3 = tex2D(ReShade::BackBuffer, texcoord + ReShade::PixelSize*fEmbossOffset*offset).rgb;

	float3 colEmboss = col1 * 2.0 - col2 - col3;
	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;
	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );
	res = lerp( colFinal, col2, luminance * luminance );

	if( bDepthCheck ){
		float depth1 = ReShade::GetLinearizedDepth(texcoord - ReShade::PixelSize*fEmbossOffset).r;
		float depth2 = ReShade::GetLinearizedDepth(texcoord);
		float depth3 = ReShade::GetLinearizedDepth(texcoord + ReShade::PixelSize*fEmbossOffset).r;
		
		if( max(abs(depth1-depth2),abs(depth3-depth2)) > fDepthCutoff ){
			res = origcolor;
		}
	}

	return float4(res, 1.0);
}

technique Depth_Emboss {
	pass Emboss
	{
		VertexShader = PostProcessVS;
		PixelShader = EmbossPS;
	}
}
Last Edit: 1 year 5 months ago by Citizenkane.
The administrator has disabled public write access.
The following user(s) said Thank You: Aelius Maximus, Insomnia