Help needed to get a working depth check to Emboss

  • Insomnia
  • Topic Author
More
6 years 11 months ago - 6 years 11 months ago #1 by Insomnia Help needed to get a working depth check to Emboss was created by Insomnia
Hey!

I'm trying to add a working depth check to the Emboss shader but I have no idea how to approach it honestly.
At the moment, something does seem to happen when turning the depth cutoff knob, but far from what I want.

This is the current code:
/*
	Full credits to the ReShade team
	Ported by Insomnia
*/

uniform float fEmbossPower <
	ui_type = "drag";
	ui_min = 0.01; ui_max = 2.0;
	ui_label = "Emboss Power";
> = 0.150;
uniform float fEmbossOffset <
	ui_type = "drag";
	ui_min = 0.1; ui_max = 5.0;
	ui_label = "Emboss Offset";
> = 1.00;
uniform float iEmbossAngle <
	ui_type = "drag";
	ui_min = 0.0; ui_max = 360.0;
	ui_label = "Emboss Angle";
> = 90.00;





//#define bEmbossDoDepthCheck 1 //-EXPERIMENTAL! If enabled, shader compares emboss samples depth to avoid artifacts at object borders.
//#define fEmbossDepthCutoff 0.0001 //-Preserves object edges from getting artifacts. If pixel depth difference of emboss samples is higher than that, pixel gets skipped.

uniform float fEmbossDepthCutoff <
	ui_label = "fEmbossDepthCutoff";
	ui_type = "drag";
	ui_min = 0.01; ui_max = 1.0;
	> = 0.2;

#include "ReShade.fxh"

float3 EmbossPass(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target
{
	float3 res = 0;
	float3 color = tex2D(ReShade::BackBuffer, texcoord).rgb;

    float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);
	float3 col1 = tex2D(ReShade::BackBuffer, texcoord - ReShade::PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = color.rgb;
	float3 col3 = tex2D(ReShade::BackBuffer, texcoord + ReShade::PixelSize*fEmbossOffset*offset).rgb;

		float depth1 = tex2D(ReShade::DepthBuffer,texcoord - ReShade::PixelSize*fEmbossOffset).rgb;
		float depth2 = tex2D(ReShade::DepthBuffer,texcoord).rgb;
		float depth3 = tex2D(ReShade::DepthBuffer,texcoord + ReShade::PixelSize*fEmbossOffset).rgb;

	
	float3 colEmboss = col1 * 2.0 - col2 - col3;

	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;

	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );

	color.xyz = lerp( colFinal, col2, luminance * luminance ).xyz;


        if(max(abs(depth1-depth2),abs(depth3-depth2)) > fEmbossDepthCutoff) color = res;


	return color;

}

technique Emboss_Tech
{
	pass Emboss
	{
		VertexShader = PostProcessVS;
		PixelShader = EmbossPass;
	}
}


Thanks for any input! :)
Last edit: 6 years 11 months ago by Insomnia.
The following user(s) said Thank You: Aelius Maximus

Please Log in or Create an account to join the conversation.

  • crosire
More
6 years 11 months ago #2 by crosire Replied by crosire on topic Help needed to get a working depth check to Emboss
I'm not quite sure what exactly you are trying to do, but the "color = res;" line looks wrong, since "res" is never initialized to anything other than zero (first line in the function). I doubt you want the picture to be black for pixels where your check succeeds, so you probably wanted to set "res" to the original input color:
float3 color = res = tex2D(ReShade::BackBuffer, texcoord).rgb;
The following user(s) said Thank You: Insomnia

Please Log in or Create an account to join the conversation.

  • Insomnia
  • Topic Author
More
6 years 11 months ago - 6 years 11 months ago #3 by Insomnia Replied by Insomnia on topic Help needed to get a working depth check to Emboss

crosire wrote: I'm not quite sure what exactly you are trying to do

Neither do I lol.

but the "color = res;" line looks wrong, since "res" is never initialized to anything other than zero (first line in the function). I doubt you want the picture to be black for pixels where your check succeeds, so you probably wanted to set "res" to the original input color:

float3 color = res = tex2D(ReShade::BackBuffer, texcoord).rgb;


Thanks, but not much happens I'm afraid.

Here's the original code:
NAMESPACE_ENTER(CFX)

#include CFX_SETTINGS_DEF

#if USE_EMBOSS

float4 PS_Emboss(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target
{
	float4 res = 0;
	float4 origcolor = tex2D(RFX_backbufferColor, texcoord);

        float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);
	float3 col1 = tex2D(RFX_backbufferColor, texcoord - RFX_PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = origcolor.rgb;
	float3 col3 = tex2D(RFX_backbufferColor, texcoord + RFX_PixelSize*fEmbossOffset*offset).rgb;

#if(bEmbossDoDepthCheck != 0)
	float depth1 = tex2D(RFX_depthColor,texcoord - RFX_PixelSize*fEmbossOffset).r;
	float depth2 = tex2D(RFX_depthColor,texcoord).r;
	float depth3 = tex2D(RFX_depthColor,texcoord + RFX_PixelSize*fEmbossOffset).r;
#endif
	
	float3 colEmboss = col1 * 2.0 - col2 - col3;

	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;

	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );

	res.xyz = lerp( colFinal, col2, luminance * luminance ).xyz;
#if(bEmbossDoDepthCheck != 0)
        if(max(abs(depth1-depth2),abs(depth3-depth2)) > fEmbossDepthCutoff) res = origcolor;
#endif
	return res;

}

technique Emboss_Tech <bool enabled = RFX_Start_Enabled; int toggle = Emboss_ToggleKey; >
{
	pass Emboss
	{
		VertexShader = RFX_VS_PostProcess;
		PixelShader = PS_Emboss;
	}
}

#endif

#include CFX_SETTINGS_UNDEF

NAMESPACE_LEAVE()

I've just been fooling around to try and add a working depth function on my own, but like you've noticed I don't know what the heck I'm doing.
Last edit: 6 years 11 months ago by Insomnia.

Please Log in or Create an account to join the conversation.

  • Aelius Maximus
More
6 years 11 months ago #4 by Aelius Maximus Replied by Aelius Maximus on topic Help needed to get a working depth check to Emboss
This definitely has potential, hope you get it working!

Please Log in or Create an account to join the conversation.

  • Insomnia
  • Topic Author
More
6 years 11 months ago #5 by Insomnia Replied by Insomnia on topic Help needed to get a working depth check to Emboss
Been trying to do... something, but I can't get it to work. I'm simply not meant for programming as I don't understand it.

Gonna drop this.

Please Log in or Create an account to join the conversation.

  • Citizenkane
More
6 years 11 months ago - 6 years 11 months ago #6 by Citizenkane Replied by Citizenkane on topic Help needed to get a working depth check to Emboss
I got you buddy ;)

I didn't touch the algorithm at all, just ported the code to ReShade 3.0. It would be faster with bDepthCheck as a preprocessor setting instead of a uniform, but this way you can enable/disable it in real time.

Good call including the original code. Was easier to use as a starting point.
#include "ReShade.fxh"

uniform float fEmbossPower <
	ui_type = "drag";
	ui_min = 0.01; ui_max = 2.0;
	ui_label = "Emboss Power";
> = 0.150;
uniform float fEmbossOffset <
	ui_type = "drag";
	ui_min = 0.1; ui_max = 5.0;
	ui_label = "Emboss Offset";
> = 1.00;
uniform float iEmbossAngle <
	ui_type = "drag";
	ui_min = 0.0; ui_max = 360.0;
	ui_label = "Emboss Angle";
> = 90.00;

uniform bool bDepthCheck <
	ui_label = "Enable Depth Check";
> = false;

uniform float fDepthCutoff <
	ui_type = "input";
	ui_label = "Depth Cutoff";
> = 0.001;

float4 EmbossPS(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target {
	float3 res;
	float3 origcolor = tex2D(ReShade::BackBuffer, texcoord).rgb;

	float2 offset;
	sincos(radians( iEmbossAngle), offset.y, offset.x);

	float3 col1 = tex2D(ReShade::BackBuffer, texcoord - ReShade::PixelSize*fEmbossOffset*offset).rgb;
	float3 col2 = origcolor;
	float3 col3 = tex2D(ReShade::BackBuffer, texcoord + ReShade::PixelSize*fEmbossOffset*offset).rgb;

	float3 colEmboss = col1 * 2.0 - col2 - col3;
	float colDot = max(0,dot(colEmboss, 0.333))*fEmbossPower;
	float3 colFinal = col2 - colDot;

	float luminance = dot( col2, float3( 0.6, 0.2, 0.2 ) );
	res = lerp( colFinal, col2, luminance * luminance );

	if( bDepthCheck ){
		float depth1 = ReShade::GetLinearizedDepth(texcoord - ReShade::PixelSize*fEmbossOffset).r;
		float depth2 = ReShade::GetLinearizedDepth(texcoord);
		float depth3 = ReShade::GetLinearizedDepth(texcoord + ReShade::PixelSize*fEmbossOffset).r;
		
		if( max(abs(depth1-depth2),abs(depth3-depth2)) > fDepthCutoff ){
			res = origcolor;
		}
	}

	return float4(res, 1.0);
}

technique Depth_Emboss {
	pass Emboss
	{
		VertexShader = PostProcessVS;
		PixelShader = EmbossPS;
	}
}
Last edit: 6 years 11 months ago by Citizenkane.
The following user(s) said Thank You: Aelius Maximus, Insomnia

Please Log in or Create an account to join the conversation.

We use cookies
We use cookies on our website. Some of them are essential for the operation of the forum. You can decide for yourself whether you want to allow cookies or not. Please note that if you reject them, you may not be able to use all the functionalities of the site.