Originally posted by DFrey:
When you draw the alpha texture into the stencil buffer, if you have alpha testing enabled and have the alpha test appropriately set, then it is possible just for the nonzero portion of the mask to be drawn. Then you can draw with the rgb texture using a stencil test
Hi DFrey. I’ve tried your suggestion about using stencil and alpha testing prior to drawing the color texture. Unfortunately that didn’t seem to work either.
According to the guide if a fragment passes the alpha test then the fragment moves onto the stencil test. However if the fragment fails the alpha test then testing stops and OpenGL movies onto the next fragment. Fine.
This is all very nice had OpenGL actually evaluated the alphas in my mask. Instead OpenGL chose to evaluate the alpha from glColor4f (line [8]) that was used to draw the quad my texture sits on. By changing glColor4f’s alpha I saw everything or nothing. Removing the glColor4f call wouldn’t have changed anything because OpenGL would then use what ever alpha was set from the most recent glColor call.
Masking with the alpha mask split from the color texture is proving to be a royal pain.
// loaded mask as a one component with internal GL_ALPHA format
glBindTexture( GL_TEXTURE_2D, maskName )
glTexImage2D( GL_TEXTURE_2D, 0, 1, width, height, 0, GL_ALPHA,
GL_UNSIGNED_BYTE, pMaskBits )
// loaded color bmp as 3 component with internal format GL_BGR_EXT
glBindTexture( GL_TEXTURE_2D, colorName )
glTexImage2D( GL_TEXTURE_2D, 0, 3, _width, _height, 0, GL_BGR_EXT,
GL_UNSIGNED_BYTE, pColorBits )
// create plane for textures to sit on
glNewList( listName, GL_COMPILE )
glBegin( GL_QUADS )
glNormal3f( 0.0f, 0.0f, 1.0f )
[8] glColor4f( 0.5f, 0.0f, 0.0f, 0.0f )
glTexCoord2fv( bottomRight ); glVertex2i( 40, -40 );
glTexCoord2fv( topRight ); glVertex2i( 40, 40 );
glTexCoord2fv( topLeft ); glVertex2i( -40, 40 );
glTexCoord2fv( bottomLeft ); glVertex2i( -40, -40 );
glEnd( )
glEndList( )
glClearStencil( 0 );
glClear( GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT );
glDisable( GL_BLEND );
// no depth testing so mask does not fight with color tex
glDisable( GL_DEPTH_TEST );
// Leave the color buffer alone
glColorMask( GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE );
// any alpha other than 0 moves onto the stencil test
glEnable( GL_ALPHA_TEST );
glAlphaFunc( GL_GREATER, 0 );
// any non-zero alpha becomes white in stencil buffer
// alphas==zero don’t make it this far and are discarded
// before getting to the stencil test.
glEnable( GL_STENCIL_TEST );
glStencilMask( 1 );
glStencilFunc( GL_ALWAYS, 1, 1 );
glStencilOp( GL_REPLACE, GL_REPLACE, GL_REPLACE );
// Draw mask
glEnable( GL_TEXTURE_2D );
glTexEnvf( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE );
glBindTexture( GL_TEXTURE_2D, maskName );
glCallList( listName );
glDisable( GL_ALPHA_TEST );
// Update color buffer where stencil is white and
// deny updates to stencil buffer.
glStencilFunc( GL_EQUAL, 1, 1 );
glStencilOp( GL_KEEP, GL_KEEP, GL_KEEP );
glEnable( GL_LIGHTING );
glEnable( GL_LIGHT0 );
glLightfv( GL_LIGHT0, GL_POSITION, lightPos );
// Draw color texture
glColorMask( GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE );
glTexEnvf( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE );
glBindTexture( GL_TEXTURE_2D, colorName );
glCallList( listName );
glDisable( GL_LIGHT0 );
glDisable( GL_LIGHTING );
glDisable( GL_STENCIL_TEST );
glDisable( GL_TEXTURE_2D );