Error trying to convert a glsl shader!

  • DevilSingh
  • Topic Author
More
3 years 11 months ago #1 by DevilSingh Error trying to convert a glsl shader! was created by DevilSingh
I'm trying to port Dogway's grade.glsl

And getting the error "implicit truncation of vector type" in "return"

Here :

float3 sRGB_to_XYZ(float3 RGB)
{
    const float3x3 m = float3x3(
    0.41239082813262940, 0.21263903379440308, 0.019330820068717003,
    0.35758435726165770, 0.71516871452331540, 0.119194783270359040,
    0.18048080801963806, 0.07219231873750687, 0.95053225755691530);

    return m * RGB;
}

Please Log in or Create an account to join the conversation.

  • prod80
More
3 years 11 months ago #2 by prod80 Replied by prod80 on topic Error trying to convert a glsl shader!

DevilSingh wrote: I'm trying to port Dogway's grade.glsl

And getting the error "implicit truncation of vector type" in "return"

Here :

float3 sRGB_to_XYZ(float3 RGB)
{
    const float3x3 m = float3x3(
    0.41239082813262940, 0.21263903379440308, 0.019330820068717003,
    0.35758435726165770, 0.71516871452331540, 0.119194783270359040,
    0.18048080801963806, 0.07219231873750687, 0.95053225755691530);

    return m * RGB;
}


yes, do
return mul( RGB, m );

this will produce column wise result for that matrix you have there (which seems to be column major). Here's both, row wise;
float3 pd80_srgb_to_xyz( float3 c )
{
    // Source: http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
    // sRGB to XYZ (D65) - Standard sRGB reference white ( 0.95047, 1.0, 1.08883 )
    const float3x3 mat = float3x3(
    0.4124564, 0.3575761, 0.1804375,
    0.2126729, 0.7151522, 0.0721750,
    0.0193339, 0.1191920, 0.9503041
    );
    return mul( mat, c );
}

float3 pd80_xyz_to_srgb( float3 c )
{
    // Source: http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
    // XYZ to sRGB (D65) - Standard sRGB reference white ( 0.95047, 1.0, 1.08883 )
    const float3x3 mat = float3x3(
    3.2404542,-1.5371385,-0.4985314,
   -0.9692660, 1.8760108, 0.0415560,
    0.0556434,-0.2040259, 1.0572252
    );
    return mul( mat, c );
}
The following user(s) said Thank You: DevilSingh

Please Log in or Create an account to join the conversation.

  • DevilSingh
  • Topic Author
More
3 years 11 months ago - 3 years 11 months ago #3 by DevilSingh Replied by DevilSingh on topic Error trying to convert a glsl shader!
Thank you! It worked.

EDIT - The shader loads now. But I'm only getting a black screen when I enable it!

Here's the shader :
uniform float g_gamma_out = 2.20;
uniform float g_gamma_in = 2.40;
uniform float g_gamma_type = 1.0;
uniform float g_crtgamut = 4.0;
uniform float g_hue_degrees = 0.0;
uniform float g_I_SHIFT = 0.0;
uniform float g_Q_SHIFT = 0.0;
uniform float g_I_MUL = 1.0;
uniform float g_Q_MUL = 1.0;
uniform float wp_temperature = 9305.0;
uniform float sat_r = 0.0;
uniform float sat_g = 0.0;
uniform float sat_b = 0.0;
uniform float g_vibr = 0.0;
uniform float g_lum = 0.0;
uniform float g_cntrst = 0.0;
uniform float g_mid = 0.5;
uniform float g_lift = 0.0;
uniform float blr = 0.0;
uniform float blg = 0.0;
uniform float blb = 0.0;
uniform float wlr = 1.0;
uniform float wlg = 1.0;
uniform float wlb = 1.0;
uniform float rg = 0.0;
uniform float rb = 0.0;
uniform float gr = 0.0;
uniform float gb = 0.0;
uniform float br = 0.0;
uniform float bg = 0.0;

#include "ReShade.fxh"

#define PI 3.1415926535897932384626433832795
#define fmod(x,y)(x-y*trunc(x/y))

float3 wp_adjust(float3 color)
{
    float temp = wp_temperature / 100.;
    float k = wp_temperature / 10000.;
    float lk = log(k);

    float3 wp = 1.;

    wp.r = (temp <= 65.) ? 1. : 0.32068362618584273 + (0.19668730877673762 * pow(k - 0.21298613432655075, - 1.5139012907556737)) + (- 0.013883432789258415 * lk);

    float mg = 1.226916242502167 + (- 1.3109482654223614 * pow(k - 0.44267061967913873, 3.) * exp(- 5.089297600846147 * (k - 0.44267061967913873))) + (0.6453936305542096 * lk);
    float pg = 0.4860175851734596 + (0.1802139719519286 * pow(k - 0.14573069517701578, - 1.397716496795082)) + (- 0.00803698899233844 * lk);
    wp.g = (temp <= 65.5) ? ((temp <= 8.) ? 0. : mg) : pg;

    wp.b = (temp <= 19.) ? 0. : (temp >= 66.) ? 1. : 1.677499032830161 + (- 0.02313594016938082 * pow(k - 1.1367244820333684, 3.) * exp(- 4.221279555918655 * (k - 1.1367244820333684))) + (1.6550275798913296 * lk);

    wp.rgb = clamp(wp.rgb, 0., 1.);

    return color * wp;
}

float3 sRGB_to_XYZ(float3 RGB)
{
    const float3x3 m = float3x3(
    0.41239082813262940, 0.21263903379440308, 0.019330820068717003,
    0.35758435726165770, 0.71516871452331540, 0.119194783270359040,
    0.18048080801963806, 0.07219231873750687, 0.95053225755691530);

    return mul(RGB,m);
}

float3 XYZtoYxy(float3 XYZ)
{
    float XYZrgb = XYZ.r+XYZ.g+XYZ.b;
    float Yxyg = (XYZrgb <= 0.0) ? 0.3805 : XYZ.r / XYZrgb;
    float Yxyb = (XYZrgb <= 0.0) ? 0.3769 : XYZ.g / XYZrgb;

    return float3(XYZ.g, Yxyg, Yxyb);
}

float3 YxytoXYZ(float3 Yxy)
{
    float Xs = Yxy.r * (Yxy.g/Yxy.b);
    float Xsz = (Yxy.r <= 0.0) ? 0.0 : 1.0;
    float3 XYZ = float3(Xsz,Xsz,Xsz) * float3(Xs, Yxy.r, (Xs/Yxy.g)-Xs-Yxy.r);

    return XYZ;
}

float3 XYZ_to_sRGB(float3 XYZ)
{
    const float3x3 m = float3x3(
     3.2409696578979490, -0.96924358606338500,  0.055630072951316833,
    -1.5373830795288086,  1.87596738338470460, -0.203976929187774660,
    -0.4986107349395752,  0.04155508056282997,  1.056971430778503400);

   return mul(XYZ,m);
}

float3 mixfix(float3 a, float3 b, float c)
{
	return (a.z < 1.0) ? lerp(a, b, c) : a;
}

float4 mixfix_v4(float4 a, float4 b, float c)
{
    return (a.z < 1.0) ? lerp(a, b, c) : a;
}

float SatMask(float color_r, float color_g, float color_b)
{
	float max_rgb = max(color_r, max(color_g, color_b));
	float min_rgb = min(color_r, min(color_g, color_b));
	float msk = clamp((max_rgb - min_rgb) / (max_rgb + min_rgb), 0.0, 1.0);

	return msk;
}

float moncurve_f(float color, float gamma, float offs)
{
	color = clamp(color, 0.0, 1.0);
	float fs = (( gamma - 1.0) / offs) * pow( offs * gamma / ( ( gamma - 1.0) * ( 1.0 + offs)), gamma);
	float xb = offs / ( gamma - 1.0);
	color = ( color > xb) ? pow( ( color + offs) / ( 1.0 + offs), gamma) : color * fs;

	return color;
}

float3 moncurve_f_f3(float3 color, float gamma, float offs)
{
	color.r = moncurve_f( color.r, gamma, offs);
	color.g = moncurve_f( color.g, gamma, offs);
	color.b = moncurve_f( color.b, gamma, offs);

	return color.rgb;
}

float moncurve_r(float color, float gamma, float offs)
{
	color = clamp(color, 0.0, 1.0);
	float yb = pow( offs * gamma / ( ( gamma - 1.0) * ( 1.0 + offs)), gamma);
	float rs = pow( ( gamma - 1.0) / offs, gamma - 1.0) * pow( ( 1.0 + offs) / gamma, gamma);
	color = ( color > yb) ? ( 1.0 + offs) * pow( color, 1.0 / gamma) - offs : color * rs;

	return color;
}

float3 moncurve_r_f3(float3 color, float gamma, float offs)
{
	color.r = moncurve_r( color.r, gamma, offs);
	color.g = moncurve_r( color.g, gamma, offs);
	color.b = moncurve_r( color.b, gamma, offs);

	return color.rgb;
}

float contrast_sigmoid(float color, float cont, float pivot)
{
	cont = pow(cont + 1.0, 3.0);
	float knee = 1.0 / (1.0 + exp(cont * pivot));
	float shldr = 1.0 / (1.0 + exp(cont * (pivot - 1.0)));
	color = (1.0 / (1.0 + exp(cont * (pivot - color))) - knee) / (shldr - knee);

	return color;
}

float contrast_sigmoid_inv(float color, float cont, float pivot)
{
	cont = pow(cont - 1.0, 3.0);
	float knee = 1.0 / (1.0 + exp (cont * pivot));
	float shldr = 1.0 / (1.0 + exp (cont * (pivot - 1.0)));
	color = pivot - log(1.0 / (color * (shldr - knee) + knee) - 1.0) / cont;

	return color;
}

float rolled_gain(float color, float gain)
{
	float gx = gain + 1.0;
	float ax = (max(0.5 - (gx / 2.0), 0.5));
	float cx = (gx > 0.0) ? (1.0 - gx + (gx / 2.0)) : abs(gx) / 2.0;
	float gain_plus = ((color * gx) > ax) ? (ax + cx * tanh((color * gx - ax) / cx)) : (color * gx);
	float ax_g = 1.0 - abs(gx);
	float gain_minus = (color > ax_g) ? (ax_g + cx * tanh((color - ax_g) / cx)) : color;
	color = (gx > 0.0) ? gain_plus : gain_minus;

	return color;
}

float4 rolled_gain_v4(float4 color, float gain)
{
	color.r = rolled_gain(color.r, gain);
	color.g = rolled_gain(color.g, gain);
	color.b = rolled_gain(color.b, gain);

	return float4(color.rgb, 1.0);
}

float3 RGB_YIQ(float3 col)
{
	float3x3 conv_mat = float3x3(
	0.299996928307425,  0.590001575542717,  0.110001496149858,
	0.599002392519453, -0.277301256521204, -0.321701135998249,
	0.213001700342824, -0.525101205289350,  0.312099504946526);

	return mul(col.rgb,conv_mat);
}

float3 YIQ_RGB(float3 col)
{
	float3x3 conv_mat = float3x3(
	1.0,  0.946882217090069,  0.623556581986143,
	1.0, -0.274787646298978, -0.635691079187380,
	1.0, -1.108545034642030,  1.709006928406470);

	return mul(col.rgb,conv_mat);
}

float3 RGB_YUV(float3 RGB)
{
	float3x3 conv_mat = float3x3(
	0.29900,  0.587000,  0.11400,
   -0.14713, -0.288860,  0.43600,
	0.61500, -0.514991, -0.10001);

	return mul(RGB.rgb,conv_mat);
}

float3 YUV_RGB(float3 YUV)
{
	float3x3 conv_mat = float3x3(
	1.000,  0.00000,  1.13983,
	1.000, -0.39465, -0.58060,
	1.000,  2.03211,  0.00000);

	return mul(YUV.rgb,conv_mat);
}

float3 PCtoTV(float3 col)
{
	col *= 255.0;
	col.x =  ((col.x * 219.0) / 255.0) +  16.0;
	col.y = (((col.y - 128.0) * 224.0) / 255.0) + 112.0;
	col.z = (((col.z - 128.0) * 224.0) / 255.0) + 112.0;

	return col.xyz / 255.0;
}

float3 TVtoPC(float3 col)
{
	col *= 255.0;
	float colx =  ((col.x -  16.0) / 219.0) * 255.0;
	float coly = (((col.y - 112.0) / 224.0) * 255.0) + 128.0;
	float colz = (((col.z - 112.0) / 224.0) * 255.0) + 128.0;

	return float3(colx,coly,colz) / 255.0;
}

static const float3x3 C_D65_Brad = 
	float3x3(
	1.0063, 0.0029, -0.0071,
	0.0036, 0.9992, -0.0024,
   -0.0013, 0.0022,  0.9645);

static const float3x3 D93_D65_Brad = 
	float3x3(
	1.0472, 0.0198, -0.0476,
	0.0250, 0.9988, -0.0160,
   -0.0090, 0.0148,  0.7659);

static const float3x3 PAL_D65_Brad = 
	float3x3(
	0.9992, -0.0005, 0.0003,
   -0.0007,  1.0005, 0.0001,
	0.0000, -0.0000, 1.0008);

static const float3x3 P22_transform = 
	float3x3(
	0.4665, 0.2566, 0.0058,
	0.3039, 0.6682, 0.1056,
	0.1800, 0.0752, 0.9776);

static const float3x3 NTSC_FCC_transform = 
	float3x3(
	0.5803, 0.2858, 0.0000,
	0.1791, 0.6055, 0.0682,
	0.1902, 0.1087, 1.0599);

static const float3x3 SMPTE_transform = 
	float3x3(
	0.3935, 0.2124, 0.0187,
	0.3653, 0.7011, 0.1119,
	0.1917, 0.0866, 0.9584);

static const float3x3 Conrad_transform = 
	float3x3(
	0.5584, 0.2858, 0.0352,
	0.2061, 0.6371, 0.0937,
	0.1859, 0.0771, 0.9602);

static const float3x3 NTSC_J_transform = 
	float3x3(
	0.3960, 0.2243, 0.0205,
	0.3120, 0.6742, 0.1281,
	0.2450, 0.1015, 1.2651);

static const float3x3 EBU_transform = 
	float3x3(
	0.4319, 0.2227, 0.0202,
	0.3412, 0.7060, 0.1294,
	0.1782, 0.0713, 0.9385);

static const float3x3 Sony20_20_transform = 
	float3x3(
	0.3863, 0.2101, 0.0216,
	0.3191, 0.6780, 0.1538,
	0.2477, 0.1118, 1.2383);

float4 GradePS(float4 position:SV_Position,float2 texcoord:TEXCOORD):SV_Target
{
	float3 source = tex2D(ReShade::BackBuffer,texcoord).rgb;
    float3 col = (g_crtgamut == 5.0) ? RGB_YUV(source) :
                 (g_crtgamut == 4.0) ? RGB_YIQ(source) :
                                PCtoTV(RGB_YIQ(source));

    float hue_radians = g_hue_degrees * (PI / 180.0);
    float hue = atan2(col.z, col.y) + hue_radians;
    float chroma = sqrt(col.z * col.z + col.y * col.y);
    col = float3(col.x, chroma * cos(hue), chroma * sin(hue));

    col.y = fmod((col.y + 1.0) + g_I_SHIFT, 2.0) - 1.0;
    col.z = fmod((col.z + 1.0) + g_Q_SHIFT, 2.0) - 1.0;

    col.z *= g_Q_MUL;
    col.y *= g_I_MUL;

    float TV_lvl = (g_crtgamut == 5.0) ? 0.0627  :
                   (g_crtgamut == 4.0) ? 0.0627  :
                                         0.0;

    col = (g_crtgamut == 5.0) ? clamp(col.xyz,float3(0.0627-TV_lvl,0.0627-0.5-TV_lvl,0.0627-0.5-TV_lvl),float3(0.92157,0.94118-0.5,0.94118-0.5)) :
							    clamp(col.xyz,float3(0.0627-TV_lvl,-0.5957-TV_lvl,-0.5226-TV_lvl),float3(0.92157,0.5957,0.5226));

    col = (g_crtgamut == 0.0) ? source       :
          (g_crtgamut == 5.0) ? YUV_RGB(col) :
          (g_crtgamut == 4.0) ? YIQ_RGB(col) :
                                YIQ_RGB(TVtoPC(col));

    float3 imgColor = (g_gamma_type == 2.0) ? moncurve_f_f3(col, g_gamma_in - 0.18, 0.1115) : (g_gamma_type == 1.0) ? moncurve_f_f3(col, g_gamma_in, 0.055) : pow(col, g_gamma_in);

    float3 Yxy = XYZtoYxy(sRGB_to_XYZ(imgColor));
    float toGamma = clamp(moncurve_r(Yxy.r, 2.40, 0.055), 0.0, 1.0);
    toGamma = (Yxy.r > 0.5) ? contrast_sigmoid_inv(toGamma, 2.3, 0.5) : toGamma;
    float sigmoid = (g_cntrst > 0.0) ? contrast_sigmoid(toGamma, g_cntrst, g_mid) : contrast_sigmoid_inv(toGamma, g_cntrst, g_mid);
    float3 contrast = float3(moncurve_f(sigmoid, 2.40, 0.055), Yxy.g, Yxy.b);
    float3 XYZsrgb = clamp(XYZ_to_sRGB(YxytoXYZ(contrast)), 0.0, 1.0);
    contrast = (g_cntrst == 0.0) ? imgColor : XYZsrgb;

    contrast += (g_lift / 20.0) * (1.0 - contrast);

    float4 screen = float4(max(contrast, 0.0), 1.0);
    float r_sat = sat_r + 1.0;
    float g_sat = sat_g + 1.0;
    float b_sat = sat_b + 1.0;

    float4x4 color = float4x4(wlr, rg,  rb,  0.0,
                              gr,  wlg, gb,  0.0,
                              br,  bg,  wlb, 0.0,
                              blr/20.0, blg/20.0, blb/20.0, 0.0);

    float4x4 adjust = float4x4((1.0 - r_sat) * 0.2126 + r_sat, (1.0 - r_sat) * 0.2126, (1.0 - r_sat) * 0.2126, 1.0,
                               (1.0 - g_sat) * 0.7152, (1.0 - g_sat) * 0.7152 + g_sat, (1.0 - g_sat) * 0.7152, 1.0,
                               (1.0 - b_sat) * 0.0722, (1.0 - b_sat) * 0.0722, (1.0 - b_sat) * 0.0722 + b_sat, 1.0,
                                0.0, 0.0, 0.0, 1.0);

    screen = clamp(rolled_gain_v4(screen, g_lum * 2.0), 0.0, 1.0);
    screen = color * screen;
    float sat_msk = (g_vibr > 0.0) ? clamp(1.0 - (SatMask(screen.r, screen.g, screen.b) * g_vibr), 0.0, 1.0) : clamp(1.0 - abs(SatMask(screen.r, screen.g, screen.b) - 1.0) * abs(g_vibr), 0.0, 1.0);
    screen = mixfix_v4(screen, adjust * screen, sat_msk);

    float3x3 m_in;

    if (g_crtgamut == 1.0) { m_in = NTSC_FCC_transform;  } else
    if (g_crtgamut == 2.0) { m_in = P22_transform;       } else
    if (g_crtgamut == 3.0) { m_in = SMPTE_transform;     } else
    if (g_crtgamut == 4.0) { m_in = NTSC_J_transform;    } else
    if (g_crtgamut == 5.0) { m_in = EBU_transform;       } else
    if (g_crtgamut == 6.0) { m_in = Sony20_20_transform; } else
    if (g_crtgamut == 7.0) { m_in = Conrad_transform;    }

    float3 gamut = (g_crtgamut == 1.0) ? (m_in*screen.rgb)*  C_D65_Brad  :
                   (g_crtgamut == 4.0) ? (m_in*screen.rgb)*D93_D65_Brad  :
                   (g_crtgamut == 5.0) ? (m_in*screen.rgb)*PAL_D65_Brad  :
                   (g_crtgamut == 6.0) ? (m_in*screen.rgb)*D93_D65_Brad  :
                                          m_in*screen.rgb;

    float3 adjusted = (g_crtgamut == 0.0) ? wp_adjust(screen.rgb) : wp_adjust(XYZ_to_sRGB(gamut));
    float3 base_luma = XYZtoYxy(gamut);
    float3 adjusted_luma = XYZtoYxy(sRGB_to_XYZ(adjusted));
    adjusted = adjusted_luma + (float3(base_luma.r, 0.0, 0.0) - float3(adjusted_luma.r, 0.0, 0.0));
    adjusted = clamp(XYZ_to_sRGB(YxytoXYZ(adjusted)), 0.0, 1.0);

    return float4(moncurve_r_f3(adjusted, g_gamma_out + 0.20, 0.055), 1.0);
}

technique Grade
{
	pass
	{
		VertexShader=PostProcessVS;
		PixelShader=GradePS;
	}
}

(This is a custom version of the glsl shader I linked to in the first post that works fine in RetroArch!)

The only changes I made was removing all the "\" in the glsl shader as ReShade didn't seem to like them, and changed :


From (this was giving me an "intrinsic overload" error) :
float hue = atan(col.z, col.y) + hue_radians;

To :
float hue = atan(col.zy) + hue_radians;
float hue = atan2(col.z, col.y) + hue_radians;
float hue = pow(col.z, col.y) + hue_radians;

None of them work.

P.S. - I should have stated it previously "I have no past coding knowledge" (other than porting some RetroArch CRT shaders to ReShade). I'm just trying to port this for a friend!
Last edit: 3 years 11 months ago by DevilSingh. Reason: Black screen!

Please Log in or Create an account to join the conversation.

  • prod80
More
3 years 11 months ago - 3 years 11 months ago #4 by prod80 Replied by prod80 on topic Error trying to convert a glsl shader!
glsl atan(x,y) is equal to hlsl atan2(y,x)

mind that x,y are reversed

so the correct translation is
float hue = atan2(col.y, col.z)+hue_radians;

Probably it will help to search for a reference guide between the function and semantic differences between glsl and hlsl

There are also more matrix transformations in there, one would possibly have to take care off, besides the sRGB to XYZ conversions. For example here, where m_in is a 3x3 matrix, which is then multiplied with color (screen.rgb)... but there's no mul() function there, like the sRGB to XYZ conversions you had above.
float3x3 m_in;

    if (g_crtgamut == 1.0) { m_in = NTSC_FCC_transform;  } else
    if (g_crtgamut == 2.0) { m_in = P22_transform;       } else
    if (g_crtgamut == 3.0) { m_in = SMPTE_transform;     } else
    if (g_crtgamut == 4.0) { m_in = NTSC_J_transform;    } else
    if (g_crtgamut == 5.0) { m_in = EBU_transform;       } else
    if (g_crtgamut == 6.0) { m_in = Sony20_20_transform; } else
    if (g_crtgamut == 7.0) { m_in = Conrad_transform;    }

    float3 gamut = (g_crtgamut == 1.0) ? (m_in*screen.rgb)*  C_D65_Brad  :
                   (g_crtgamut == 4.0) ? (m_in*screen.rgb)*D93_D65_Brad  :
                   (g_crtgamut == 5.0) ? (m_in*screen.rgb)*PAL_D65_Brad  :
                   (g_crtgamut == 6.0) ? (m_in*screen.rgb)*D93_D65_Brad  :
                                          m_in*screen.rgb;

Also, if you're on discord it's probably easier to just take the conversation there in case you have questions... avoids hours of delays between question and answer :)
Last edit: 3 years 11 months ago by prod80.
The following user(s) said Thank You: DevilSingh

Please Log in or Create an account to join the conversation.

We use cookies
We use cookies on our website. Some of them are essential for the operation of the forum. You can decide for yourself whether you want to allow cookies or not. Please note that if you reject them, you may not be able to use all the functionalities of the site.
Ok Decline
More information