HQX and Median Filters

// hq4x filter
// Ripped from https://github.com/libretro/common-shaders/blob/master/hqx/hq4x.cg

float4 PS_HQ4X(float4 pos : SV_Position, float2 uv : TEXCOORD) : SV_Target
{
	float mx = 1.0; // start smoothing wt.
	const float k = -1.10; // wt. decrease factor
	const float max_w = 0.75; // max filter weigth
	const float min_w = 0.03; // min filter weigth
	const float lum_add = 0.33; // effects smoothing

	float4 color = tex2D(RFX_backbufferColor, uv);
	float3 c = color.xyz;

	float x = 0.5 * (1.0 / 256.0);
	float y = 0.5 * (1.0 / 32.0);

	const float3 dt = 1.0*float3(1.0, 1.0, 1.0);

	float2 dg1 = float2( x, y);
	float2 dg2 = float2(-x, y);

	float2 sd1 = dg1*0.5;
	float2 sd2 = dg2*0.5;

	float2 ddx = float2(x,0.0);
	float2 ddy = float2(0.0,y);

	float4 t1 = float4(uv-sd1,uv-ddy);
	float4 t2 = float4(uv-sd2,uv+ddx);
	float4 t3 = float4(uv+sd1,uv+ddy);
	float4 t4 = float4(uv+sd2,uv-ddx);
	float4 t5 = float4(uv-dg1,uv-dg2);
	float4 t6 = float4(uv+dg1,uv+dg2);

	float3 i1 = tex2D(RFX_backbufferColor, t1.xy).xyz;
	float3 i2 = tex2D(RFX_backbufferColor, t2.xy).xyz;
	float3 i3 = tex2D(RFX_backbufferColor, t3.xy).xyz;
	float3 i4 = tex2D(RFX_backbufferColor, t4.xy).xyz;

	float3 o1 = tex2D(RFX_backbufferColor, t5.xy).xyz;
	float3 o3 = tex2D(RFX_backbufferColor, t6.xy).xyz;
	float3 o2 = tex2D(RFX_backbufferColor, t5.zw).xyz;
	float3 o4 = tex2D(RFX_backbufferColor, t6.zw).xyz;

	float3 s1 = tex2D(RFX_backbufferColor, t1.zw).xyz;
	float3 s2 = tex2D(RFX_backbufferColor, t2.zw).xyz;
	float3 s3 = tex2D(RFX_backbufferColor, t3.zw).xyz;
	float3 s4 = tex2D(RFX_backbufferColor, t4.zw).xyz;

	float ko1 = dot(abs(o1-c),dt);
	float ko2 = dot(abs(o2-c),dt);
	float ko3 = dot(abs(o3-c),dt);
	float ko4 = dot(abs(o4-c),dt);

	float k1=min(dot(abs(i1-i3),dt),max(ko1,ko3));
	float k2=min(dot(abs(i2-i4),dt),max(ko2,ko4));

	float w1 = k2; if(ko3<ko1) w1*=ko3/ko1;
	float w2 = k1; if(ko4<ko2) w2*=ko4/ko2;
	float w3 = k2; if(ko1<ko3) w3*=ko1/ko3;
	float w4 = k1; if(ko2<ko4) w4*=ko2/ko4;

	c=(w1*o1+w2*o2+w3*o3+w4*o4+0.001*c)/(w1+w2+w3+w4+0.001);
	w1 = k*dot(abs(i1-c)+abs(i3-c),dt)/(0.125*dot(i1+i3,dt)+lum_add);
	w2 = k*dot(abs(i2-c)+abs(i4-c),dt)/(0.125*dot(i2+i4,dt)+lum_add);
	w3 = k*dot(abs(s1-c)+abs(s3-c),dt)/(0.125*dot(s1+s3,dt)+lum_add);
	w4 = k*dot(abs(s2-c)+abs(s4-c),dt)/(0.125*dot(s2+s4,dt)+lum_add);

	w1 = clamp(w1+mx,min_w,max_w);
	w2 = clamp(w2+mx,min_w,max_w);
	w3 = clamp(w3+mx,min_w,max_w);
	w4 = clamp(w4+mx,min_w,max_w);

	return float4((w1*(i1+i3)+w2*(i2+i4)+w3*(s1+s3)+w4*(s2+s4)+c)/(2.0*(w1+w2+w3+w4)+1.0), 1.0);
}

technique HQ4X_Tech < enabled = RFX_Start_Enabled; >
{
	pass
	{
		VertexShader = RFX_VS_PostProcess;
		PixelShader = PS_HQ4X;
	}
}

	float x = 0.5 * (1.0 / Width);
	float y = 0.5 * (1.0 / Hight);

float x = 0.5 * BUFFER_RCP_WIDTH;
float y = 0.5 * BUFFER_RCP_HEIGHT;

/**************************************************************************
 * Copyright (C) 2003 Maxim Stepin ( maxst@hiend3d.com )    // HQnX
 * Copyright (C) 2010 Cameron Zemek ( grom@zeminvaders.net) // HQnX
 * Copyright (C) 2011 Francois Gannaz <mytskine@gmail.com>  // HQnX
 * Copyright (C) 2014 Jules Blok ( jules@aerix.nl )         // Armada651
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 **************************************************************************
 *
 * HQnX Initinal port on ReShade by kingeric1992 .Feb/1/16
 *
 * check 
 *      https://code.google.com/archive/p/hqx/                      for original C implementation
 *      https://github.com/libretro/common-shaders/tree/master/hqx  for Cg/HLSL implementation by Armada651
 *
 **************************************************************************/
 
//settings
#define HQnX_ToggleKey (unbound)    //HQnX toggle key
#define HQnX_Scale 4                //set scale 2|3|4
#define HQnX_InputTex InputTexture  //set input texture
#define HQnX_SrcScaleX  640         //source scale X
#define HQnX_SrcScaleY  480         //source scale Y
float3  HQnX_YUVoffset    = {0, 0.5, 0.5};            //or use define
float3  HQnX_YUVthreshold = {0.1882, 0.0275, 0.0235}; //or use define

//textures & samplers.
//lut tex: .rgba == {center weight, diagonal weight, horizontal weight, vertical weight}
texture HQ4xTex < source = "ReShade/HQ4xLut.png"; > { Width = 256; Height = 256; Format = RGBA8F; };
texture HQ3xTex < source = "ReShade/HQ3xLut.png"; > { Width = 256; Height = 144; Format = RGBA8F; };
texture HQ2xTex < source = "ReShade/HQ2xLut.png"; > { Width = 256; Height = 64;  Format = RGBA8F; };

sampler HQ4xLutColor { Texture = HQ4xTex;       MinFilter = POINT; MagFilter = POINT;};
sampler HQ3xLutColor { Texture = HQ3xTex;       MinFilter = POINT; MagFilter = POINT;};
sampler HQ2xLutColor { Texture = HQ2xTex;       MinFilter = POINT; MagFilter = POINT;};
sampler HQnXSrcColor { Texture = HQnX_InputTex; MinFilter = POINT; MagFilter = POINT;};

#define HQnXLutColor(a) HQ##a##xLutColor

float3x3 yuv =
{
    0.299,  0.587,  0.114,
   -0.169, -0.331,  0.5,
      0.5, -0.419, -0.081
};

bool Diff(float3 yuv1, float3 yuv2) {
    bool3 res = abs((yuv1 + HQnX_YUVoffset) - (yuv2 + HQnX_YUVoffset)) > HQnX_YUVthreshold;
    return res.x || res.y || res.z;
}

struct VS_OUTPUT {
    float4 position  : POSITION;
    float4 texCoord0 : TEXCOORD0;
    float4 texCoord1 : TEXCOORD1;
    float4 texCoord2 : TEXCOORD2;
};

VS_OUTPUT VS_HQnX(in uint id : SV_VertexID)
{
    VS_OUTPUT OUT;

    float2 coord = 0;
    coord.x = (id == 2) ? 2.0 : 0.0;
    coord.y = (id == 1) ? 2.0 : 0.0;

    float4 offset = {- 1 / HQnX_SrcScaleX, 0, 1 / HQnX_SrcScaleX, 1 / HQnX_SrcScaleY}; //  Src Pixel Size
    OUT.position  = float4(coord * float2(2.0, -2.0) + float2(-1.0, 1.0), 0.0, 1.0);
    OUT.texCoord0 = coord.xxxy + offset.xyzw;
    OUT.texCoord1 = coord.xxxy + offset.xyzy;
    OUT.texCoord2 = coord.xxxy - offset.zyxw;
    return OUT;
}

//*  use point sampler
//** only support upscale x2, x3, x4.
float4 PS_HQnX(in VS_OUTPUT IN) : COLOR
{
    float4   offset = {- 1 / HQnX_SrcScaleX, 0, 1 / HQnX_SrcScaleX, 1 / HQnX_SrcScaleY}; //  Src Pixel Size
    float2   index  = frac(IN.texCoord / offset.zw);
    float2   dir    = sign(index - 0.5);
    float4x3 pixels = //4 rows, 3 column
    {
        tex2D(HQnXSrcColor, IN.texCoord1.yw                  ).rgb, //center
        tex2D(HQnXSrcColor, IN.texCoord1.yw + dir * offset.zw).rgb, //diagonal
        tex2D(HQnXSrcColor, IN.texCoord1.yw + dir * offset.zy).rgb, //horizontal
        tex2D(HQnXSrcColor, IN.texCoord1.yw + dir * offset.yw).rgb  //vertical
    };

    float3 w[9];                                               //   +------+------+------+
    w[1] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord0.xw).rgb); //   |      |      |      |
    w[2] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord0.yw).rgb); //   | w[1] | w[2] | w[3] |
    w[3] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord0.zw).rgb); //   +------+------+------+
    w[4] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord1.xw).rgb); //   |      |      |      |
    w[0] = mul(yuv, pixels[0]);                                //   | w[4] | w[0] | w[5] |
    w[5] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord1.zw).rgb); //   +------+------+------+
    w[6] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord2.xw).rgb); //   |      |      |      |
    w[7] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord2.yw).rgb); //   | w[6] | w[7] | w[8] |
    w[8] = mul(yuv, tex2D(HQnXSrcColor, IN.texCoord2.zw).rgb); //   +------+------+------+

    //use LUT 256 cases * (16 if else * (n * n) bias) to replace switch table, index to get lut offset
    //check https://code.google.com/archive/p/hqx/ cpu source on switch table.
    index    = floor(index * HQnX_Scale);
    index.y  = index.x + index.y * HQnX_Scale;
    index.y += (Diff(w[4], w[2]) + (Diff(w[2], w[5]) << 1) + (Diff(w[7], w[4]) << 2) + (Diff(w[5], w[7]) << 3)) * (HQnX_Scale * HQnX_Scale);
    index.x  = Diff(w[0], w[1]);

    for(int i=1; i<8; i++)
        index.x += Diff(w[0], w[i+1]) << i;

    index   = (index + 0.5) / float2(256.0, 16.0 * (HQnX_Scale * HQnX_Scale));

//or use linear sampling to colapse sample quad (bypass float4x3 pixels) similar to optimized bicubic filter
/*
    float4 weight = tex2D(HQnXLutColor(HQnX_Scale), index);
    index.y = (weight.y + weight.w ) / dot(weight, 1);
    index.x = lerp( weight.z / (weight.x + weight.z), weight.w / (weight.y + weight.w), index.y);
    return tex2D(decal, (floor(IN.texCoord / offset.zw) + 0.5 + index * dir) * offset.zw));
*/
    float4 weight = tex2D(HQnXLutColor(HQnX_Scale), index);
    return float4(mul(weight / dot(weight, 1), pixels), 1.0);
}

technique HQnX < enabled = RFX_Start_Enabled; toggle = HQnX_ToggleKey; >
{
	pass 	{
		VertexShader = VS_HQnX;
		PixelShader  = PS_HQnX;
	}
}