Image-Based Rendering in the Gradient Domain

Data Term Rendering

We store each depth map in a vertex array of the format below. Each image pixel is stored using 4 vertices, one for each pixel corner. All four vertices of a pixel have the same position and uv, the 'type' variable runs from 0-3 to indicate which corner the vertex specifies.

struct VertexType
{
    float posx, posy, posz; // 3D position
    Byte type; // pixel corner 0-3, see Figure 5a in the paper
    UInt16 tx, ty; // pixel coordinate
};

The cameras can be set up as follows:

// projection matrix
Mtx4x4 P = perspectiveHorz(renderData.ofov, displayAspect, 0.1, 1000);

// initial modelview matrices
Mtx4x4 Mo = Mtx4x4(cam.right.x, cam.right.y, cam.right.z, cam.right*-cam.pos,
                   cam.up.x, cam.up.y, cam.up.z, cam.up*-cam.pos,
                   -cam.front.x, -cam.front.y, -cam.front.z, -cam.front*-cam.pos,
                   0.0, 0.0, 0.0, 1.0);

Mtx4x4 Mn = Mtx4x4(view.right.x, view.right.y, view.right.z, view.right*-view.pos,
                   view.up.x, view.up.y, view.up.z, view.up*-view.pos,
                   -view.front.x, -view.front.y, -view.front.z, -view.front*-view.pos,
                   0.0, 0.0, 0.0, 1.0);
				   
// rectified modelview matrices
Vec3 right = (cam.pos - view.pos).Unit();
Vec3 up = cam.front.Cross(right).Unit();
Vec3 front = right.Cross(up).Unit();

Mo = Mtx4x4(right.x, right.y, right.z, right*-cam.pos,
            up.x, up.y, up.z, up*-cam.pos,
            -front.x, -front.y, -front.z, -front*-cam.pos,
            0.0, 0.0, 0.0, 1.0);

Mn = Mtx4x4(right.x, right.y, right.z, right*-view.pos,
            up.x, up.y, up.z, up*-view.pos,
            -front.x, -front.y, -front.z, -front*-view.pos,
            0.0, 0.0, 0.0, 1.0);

// rectifying homography
Mtx3x4 To = Mtx3x4(P) * Mo;
Mtx3x4 Tn = Mtx3x4(P) * Mn;
Mtx3x3 H = To * Tn.T() * (Tn*Tn.T()).Inv();

The following code shows the vertex shader for rendering the horizontal gradients. The shader for vertical gradients is similar.

#version 130

in vec3 pos; // vertex position
in int type; // which corner of the pixel quad is this vertex? (see Figure 5a in the paper)
in ivec2 uv; // vertex texture coordinates

out vec4 color;

uniform sampler2D tex;
uniform float camFov, camAspect, camW, camH; // camera field of view, aspect ratio, width, height
uniform vec3 camPos, camFront, camRight, camUp // camera position, front-, right-, up-vectors;
uniform float displayWidth, displayHeight; // viewport width, height
uniform mat3 H;
uniform mat4 To, Tn;
uniform vec2 screenSpaceRightVector;
uniform float gradThresh;

void main(void)
{
    float maxy = tan(camFov/2.0);
    float maxx = maxy * camAspect;

    float ofsy = (type >= 2 ? 1.0 : 0.0);

    vec3 ray = camFront + camRight*maxx*(-1.0+2.0*(uv.x+1.0)/camW)
                        - camUp*maxy*(-1.0+2.0*(uv.y+ofsy)/camH);
    float zdist = dot((pos - camPos), camFront);

    vec4 vpos = vec4(camPos + ray*zdist, 1.0);

    vec4 proj4Neutral = To * vpos;
    vec4 proj4Current = Tn * vpos;
    
    vec3 proj3Neutral = H * vec3(proj4Neutral.x/proj4Neutral.w, proj4Neutral.y/proj4Neutral.w, 1.0);
    vec3 proj3Current = H * vec3(proj4Current.x/proj4Current.w, proj4Current.y/proj4Current.w, 1.0);

    vec2 proj2Neutral = vec2(proj3Neutral.x/proj3Neutral.z, proj3Neutral.y/proj3Neutral.z);
    vec2 proj2Current = vec2(proj3Current.x/proj3Current.z, proj3Current.y/proj3Current.z);

    float gradSign = (dot(proj2Current - proj2Neutral, screenSpaceRightVector) <= 0 ? 1.0 : -1.0);

    vec2 proj2 = (type >= 1 && type <= 2 ? proj2Current : proj2Neutral);    

    gl_Position = vec4(proj2, 0.0, 1.0);

    vec4 col0 = texelFetch(tex, uv, 0);
    vec4 col1 = texelFetch(tex, ivec2(uv.x+1, uv.y), 0);

    color = vec4((col1 - col0).xyz * gradSign, 1.0);
}

Back to previous page