/*
* Copyright © 2011 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "intel_batchbuffer.h"
#include "intel_mipmap_tree.h"
#include "intel_fbo.h"
#include "intel_resolve_map.h"
#include "brw_context.h"
#include "brw_state.h"
#include "brw_defines.h"
#include "brw_wm.h"
#include "main/framebuffer.h"
/**
* Helper function to emit depth related command packets.
*/
static void
emit_depth_packets(struct brw_context *brw,
struct intel_mipmap_tree *depth_mt,
uint32_t depthbuffer_format,
uint32_t depth_surface_type,
bool depth_writable,
struct intel_mipmap_tree *stencil_mt,
bool stencil_writable,
bool hiz,
uint32_t width,
uint32_t height,
uint32_t depth,
uint32_t lod,
uint32_t min_array_element)
{
uint32_t mocs_wb = brw->gen >= 9 ? SKL_MOCS_WB : BDW_MOCS_WB;
/* Skip repeated NULL depth/stencil emits (think 2D rendering). */
if (!depth_mt && !stencil_mt && brw->no_depth_or_stencil) {
assert(brw->hw_ctx);
return;
}
brw_emit_depth_stall_flushes(brw);
/* _NEW_BUFFERS, _NEW_DEPTH, _NEW_STENCIL */
BEGIN_BATCH(8);
OUT_BATCH(GEN7_3DSTATE_DEPTH_BUFFER << 16 | (8 - 2));
OUT_BATCH(depth_surface_type << 29 |
(depth_writable ? (1 << 28) : 0) |
(stencil_mt != NULL && stencil_writable) << 27 |
(hiz ? 1 : 0) << 22 |
depthbuffer_format << 18 |
(depth_mt ? depth_mt->pitch - 1 : 0));
if (depth_mt) {
OUT_RELOC64(depth_mt->bo,
I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, 0);
} else {
OUT_BATCH(0);
OUT_BATCH(0);
}
OUT_BATCH(((width - 1) << 4) | ((height - 1) << 18) | lod);
OUT_BATCH(((depth - 1) << 21) | (min_array_element << 10) | mocs_wb);
OUT_BATCH(0);
OUT_BATCH(((depth - 1) << 21) | (depth_mt ? depth_mt->qpitch >> 2 : 0));
ADVANCE_BATCH();
if (!hiz) {
BEGIN_BATCH(5);
OUT_BATCH(GEN7_3DSTATE_HIER_DEPTH_BUFFER << 16 | (5 - 2));
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
ADVANCE_BATCH();
} else {
assert(depth_mt);
BEGIN_BATCH(5);
OUT_BATCH(GEN7_3DSTATE_HIER_DEPTH_BUFFER << 16 | (5 - 2));
OUT_BATCH((depth_mt->hiz_buf->aux_base.pitch - 1) | mocs_wb << 25);
OUT_RELOC64(depth_mt->hiz_buf->aux_base.bo,
I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(depth_mt->hiz_buf->aux_base.qpitch >> 2);
ADVANCE_BATCH();
}
if (stencil_mt == NULL) {
BEGIN_BATCH(5);
OUT_BATCH(GEN7_3DSTATE_STENCIL_BUFFER << 16 | (5 - 2));
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
ADVANCE_BATCH();
} else {
BEGIN_BATCH(5);
OUT_BATCH(GEN7_3DSTATE_STENCIL_BUFFER << 16 | (5 - 2));
/* The stencil buffer has quirky pitch requirements. From the Graphics
* BSpec: vol2a.11 3D Pipeline Windower > Early Depth/Stencil Processing
* > Depth/Stencil Buffer State > 3DSTATE_STENCIL_BUFFER [DevIVB+],
* field "Surface Pitch":
*
* The pitch must be set to 2x the value computed based on width, as
* the stencil buffer is stored with two rows interleaved.
*
* (Note that it is not 100% clear whether this intended to apply to
* Gen7; the BSpec flags this comment as "DevILK,DevSNB" (which would
* imply that it doesn't), however the comment appears on a "DevIVB+"
* page (which would imply that it does). Experiments with the hardware
* indicate that it does.
*/
OUT_BATCH(HSW_STENCIL_ENABLED | mocs_wb << 22 |
(2 * stencil_mt->pitch - 1));
OUT_RELOC64(stencil_mt->bo,
I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, 0);
OUT_BATCH(stencil_mt ? stencil_mt->qpitch >> 2 : 0);
ADVANCE_BATCH();
}
BEGIN_BATCH(3);
OUT_BATCH(GEN7_3DSTATE_CLEAR_PARAMS << 16 | (3 - 2));
OUT_BATCH(depth_mt ? depth_mt->depth_clear_value : 0);
OUT_BATCH(1);
ADVANCE_BATCH();
brw->no_depth_or_stencil = !depth_mt && !stencil_mt;
}
/* Awful vtable-compatible function; should be cleaned up in the future. */
void
gen8_emit_depth_stencil_hiz(struct brw_context *brw,
struct intel_mipmap_tree *depth_mt,
uint32_t depth_offset,
uint32_t depthbuffer_format,
uint32_t depth_surface_type,
struct intel_mipmap_tree *stencil_mt,
bool hiz, bool separate_stencil,
uint32_t width, uint32_t height,
uint32_t tile_x, uint32_t tile_y)
{
struct gl_context *ctx = &brw->ctx;
struct gl_framebuffer *fb = ctx->DrawBuffer;
uint32_t surftype;
unsigned int depth = 1;
unsigned int min_array_element;
GLenum gl_target = GL_TEXTURE_2D;
unsigned int lod;
const struct intel_mipmap_tree *mt = depth_mt ? depth_mt : stencil_mt;
const struct intel_renderbuffer *irb = NULL;
const struct gl_renderbuffer *rb = NULL;
irb = intel_get_renderbuffer(fb, BUFFER_DEPTH);
if (!irb)
irb = intel_get_renderbuffer(fb, BUFFER_STENCIL);
rb = (struct gl_renderbuffer *) irb;
if (rb) {
depth = MAX2(irb->layer_count, 1);
if (rb->TexImage)
gl_target = rb->TexImage->TexObject->Target;
}
switch (gl_target) {
case GL_TEXTURE_CUBE_MAP_ARRAY:
case GL_TEXTURE_CUBE_MAP:
/* The PRM claims that we should use BRW_SURFACE_CUBE for this
* situation, but experiments show that gl_Layer doesn't work when we do
* this. So we use BRW_SURFACE_2D, since for rendering purposes this is
* equivalent.
*/
surftype = BRW_SURFACE_2D;
depth *= 6;
break;
case GL_TEXTURE_3D:
assert(mt);
depth = MAX2(mt->logical_depth0, 1);
surftype = translate_tex_target(gl_target);
break;
case GL_TEXTURE_1D_ARRAY:
case GL_TEXTURE_1D:
if (brw->gen >= 9) {
/* WaDisable1DDepthStencil. Skylake+ doesn't support 1D depth
* textures but it does allow pretending it's a 2D texture
* instead.
*/
surftype = BRW_SURFACE_2D;
break;
}
/* fallthrough */
default:
surftype = translate_tex_target(gl_target);
break;
}
min_array_element = irb ? irb->mt_layer : 0;
lod = irb ? irb->mt_level - irb->mt->first_level : 0;
if (mt) {
width = mt->logical_width0;
height = mt->logical_height0;
}
emit_depth_packets(brw, depth_mt, brw_depthbuffer_format(brw), surftype,
brw_depth_writes_enabled(brw),
stencil_mt, ctx->Stencil._WriteEnabled,
hiz, width, height, depth, lod, min_array_element);
}
/**
* Should we set the PMA FIX ENABLE bit?
*
* To avoid unnecessary depth related stalls, we need to set this bit.
* However, there is a very complicated formula which governs when it
* is legal to do so. This function computes that.
*
* See the documenation for the CACHE_MODE_1 register, bit 11.
*/
static bool
pma_fix_enable(const struct brw_context *brw)
{
const struct gl_context *ctx = &brw->ctx;
/* BRW_NEW_FS_PROG_DATA */
const struct brw_wm_prog_data *wm_prog_data =
brw_wm_prog_data(brw->wm.base.prog_data);
/* _NEW_BUFFERS */
struct intel_renderbuffer *depth_irb =
intel_get_renderbuffer(ctx->DrawBuffer, BUFFER_DEPTH);
/* 3DSTATE_WM::ForceThreadDispatch is never used. */
const bool wm_force_thread_dispatch = false;
/* 3DSTATE_RASTER::ForceSampleCount is never used. */
const bool raster_force_sample_count_nonzero = false;
/* _NEW_BUFFERS:
* 3DSTATE_DEPTH_BUFFER::SURFACE_TYPE != NULL &&
* 3DSTATE_DEPTH_BUFFER::HIZ Enable
*/
const bool hiz_enabled = depth_irb && intel_renderbuffer_has_hiz(depth_irb);
/* 3DSTATE_WM::Early Depth/Stencil Control != EDSC_PREPS (2). */
const bool edsc_not_preps = !wm_prog_data->early_fragment_tests;
/* 3DSTATE_PS_EXTRA::PixelShaderValid is always true. */
const bool pixel_shader_valid = true;
/* !(3DSTATE_WM_HZ_OP::DepthBufferClear ||
* 3DSTATE_WM_HZ_OP::DepthBufferResolve ||
* 3DSTATE_WM_HZ_OP::Hierarchical Depth Buffer Resolve Enable ||
* 3DSTATE_WM_HZ_OP::StencilBufferClear)
*
* HiZ operations are done outside of the normal state upload, so they're
* definitely not happening now.
*/
const bool in_hiz_op = false;
/* _NEW_DEPTH:
* DEPTH_STENCIL_STATE::DepthTestEnable
*/
const bool depth_test_enabled = depth_irb && ctx->Depth.Test;
/* _NEW_DEPTH:
* 3DSTATE_WM_DEPTH_STENCIL::DepthWriteEnable &&
* 3DSTATE_DEPTH_BUFFER::DEPTH_WRITE_ENABLE.
*/
const bool depth_writes_enabled = brw_depth_writes_enabled(brw);
/* _NEW_STENCIL:
* !DEPTH_STENCIL_STATE::Stencil Buffer Write Enable ||
* !3DSTATE_DEPTH_BUFFER::Stencil Buffer Enable ||
* !3DSTATE_STENCIL_BUFFER::Stencil Buffer Enable
*/
const bool stencil_writes_enabled = ctx->Stencil._WriteEnabled;
/* 3DSTATE_PS_EXTRA::Pixel Shader Computed Depth Mode != PSCDEPTH_OFF */
const bool ps_computes_depth =
wm_prog_data->computed_depth_mode != BRW_PSCDEPTH_OFF;
/* BRW_NEW_FS_PROG_DATA: 3DSTATE_PS_EXTRA::PixelShaderKillsPixels
* BRW_NEW_FS_PROG_DATA: 3DSTATE_PS_EXTRA::oMask Present to RenderTarget
* _NEW_MULTISAMPLE: 3DSTATE_PS_BLEND::AlphaToCoverageEnable
* _NEW_COLOR: 3DSTATE_PS_BLEND::AlphaTestEnable
* _NEW_BUFFERS: 3DSTATE_PS_BLEND::AlphaTestEnable
* 3DSTATE_PS_BLEND::AlphaToCoverageEnable
*
* 3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable is always false.
* 3DSTATE_WM::ForceKillPix != ForceOff is always true.
*/
const bool kill_pixel =
wm_prog_data->uses_kill ||
wm_prog_data->uses_omask ||
_mesa_is_alpha_test_enabled(ctx) ||
_mesa_is_alpha_to_coverage_enabled(ctx);
/* The big formula in CACHE_MODE_1::NP PMA FIX ENABLE. */
return !wm_force_thread_dispatch &&
!raster_force_sample_count_nonzero &&
hiz_enabled &&
edsc_not_preps &&
pixel_shader_valid &&
!in_hiz_op &&
depth_test_enabled &&
(ps_computes_depth ||
(kill_pixel && (depth_writes_enabled || stencil_writes_enabled)));
}
void
gen8_write_pma_stall_bits(struct brw_context *brw, uint32_t pma_stall_bits)
{
struct gl_context *ctx = &brw->ctx;
/* If we haven't actually changed the value, bail now to avoid unnecessary
* pipeline stalls and register writes.
*/
if (brw->pma_stall_bits == pma_stall_bits)
return;
brw->pma_stall_bits = pma_stall_bits;
/* According to the PIPE_CONTROL documentation, software should emit a
* PIPE_CONTROL with the CS Stall and Depth Cache Flush bits set prior
* to the LRI. If stencil buffer writes are enabled, then a Render Cache
* Flush is also necessary.
*/
const uint32_t render_cache_flush =
ctx->Stencil._WriteEnabled ? PIPE_CONTROL_RENDER_TARGET_FLUSH : 0;
brw_emit_pipe_control_flush(brw,
PIPE_CONTROL_CS_STALL |
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
render_cache_flush);
/* CACHE_MODE_1 is a non-privileged register. */
BEGIN_BATCH(3);
OUT_BATCH(MI_LOAD_REGISTER_IMM | (3 - 2));
OUT_BATCH(GEN7_CACHE_MODE_1);
OUT_BATCH(GEN8_HIZ_PMA_MASK_BITS | pma_stall_bits);
ADVANCE_BATCH();
/* After the LRI, a PIPE_CONTROL with both the Depth Stall and Depth Cache
* Flush bits is often necessary. We do it regardless because it's easier.
* The render cache flush is also necessary if stencil writes are enabled.
*/
brw_emit_pipe_control_flush(brw,
PIPE_CONTROL_DEPTH_STALL |
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
render_cache_flush);
}
static void
gen8_emit_pma_stall_workaround(struct brw_context *brw)
{
uint32_t bits = 0;
if (brw->gen >= 9)
return;
if (pma_fix_enable(brw))
bits |= GEN8_HIZ_NP_PMA_FIX_ENABLE | GEN8_HIZ_NP_EARLY_Z_FAILS_DISABLE;
gen8_write_pma_stall_bits(brw, bits);
}
const struct brw_tracked_state gen8_pma_fix = {
.dirty = {
.mesa = _NEW_BUFFERS |
_NEW_COLOR |
_NEW_DEPTH |
_NEW_MULTISAMPLE |
_NEW_STENCIL,
.brw = BRW_NEW_BLORP |
BRW_NEW_FS_PROG_DATA,
},
.emit = gen8_emit_pma_stall_workaround
};
/**
* Emit packets to perform a depth/HiZ resolve or fast depth/stencil clear.
*
* See the "Optimized Depth Buffer Clear and/or Stencil Buffer Clear" section
* of the hardware documentation for details.
*/
void
gen8_hiz_exec(struct brw_context *brw, struct intel_mipmap_tree *mt,
unsigned int level, unsigned int layer, enum blorp_hiz_op op)
{
if (op == BLORP_HIZ_OP_NONE)
return;
/* Disable the PMA stall fix since we're about to do a HiZ operation. */
if (brw->gen == 8)
gen8_write_pma_stall_bits(brw, 0);
assert(mt->first_level == 0);
assert(mt->logical_depth0 >= 1);
/* If we're operating on LOD 0, align to 8x4 to meet the alignment
* requirements for most HiZ operations. Otherwise, use the actual size
* to allow the hardware to calculate the miplevel offsets correctly.
*/
uint32_t surface_width = ALIGN(mt->logical_width0, level == 0 ? 8 : 1);
uint32_t surface_height = ALIGN(mt->logical_height0, level == 0 ? 4 : 1);
/* From the documentation for 3DSTATE_WM_HZ_OP: "3DSTATE_MULTISAMPLE packet
* must be used prior to this packet to change the Number of Multisamples.
* This packet must not be used to change Number of Multisamples in a
* rendering sequence."
*/
if (brw->num_samples != mt->num_samples) {
gen8_emit_3dstate_multisample(brw, mt->num_samples);
brw->NewGLState |= _NEW_MULTISAMPLE;
}
/* The basic algorithm is:
* - If needed, emit 3DSTATE_{DEPTH,HIER_DEPTH,STENCIL}_BUFFER and
* 3DSTATE_CLEAR_PARAMS packets to set up the relevant buffers.
* - If needed, emit 3DSTATE_DRAWING_RECTANGLE.
* - Emit 3DSTATE_WM_HZ_OP with a bit set for the particular operation.
* - Do a special PIPE_CONTROL to trigger an implicit rectangle primitive.
* - Emit 3DSTATE_WM_HZ_OP with no bits set to return to normal rendering.
*/
emit_depth_packets(brw, mt,
brw_depth_format(brw, mt->format),
BRW_SURFACE_2D,
true, /* depth writes */
NULL, false, /* no stencil for now */
true, /* hiz */
surface_width,
surface_height,
mt->logical_depth0,
level,
layer); /* min_array_element */
/* Depth buffer clears and HiZ resolves must use an 8x4 aligned rectangle.
* Note that intel_miptree_level_enable_hiz disables HiZ for miplevels > 0
* which aren't 8x4 aligned, so expanding the size is safe - it'll just
* draw into empty padding space.
*/
unsigned rect_width = ALIGN(minify(mt->logical_width0, level), 8);
unsigned rect_height = ALIGN(minify(mt->logical_height0, level), 4);
BEGIN_BATCH(4);
OUT_BATCH(_3DSTATE_DRAWING_RECTANGLE << 16 | (4 - 2));
OUT_BATCH(0);
OUT_BATCH(((rect_width - 1) & 0xffff) | ((rect_height - 1) << 16));
OUT_BATCH(0);
ADVANCE_BATCH();
/* Emit 3DSTATE_WM_HZ_OP to override pipeline state for the particular
* resolve or clear operation we want to perform.
*/
uint32_t dw1 = 0;
switch (op) {
case BLORP_HIZ_OP_DEPTH_RESOLVE:
dw1 |= GEN8_WM_HZ_DEPTH_RESOLVE;
break;
case BLORP_HIZ_OP_HIZ_RESOLVE:
dw1 |= GEN8_WM_HZ_HIZ_RESOLVE;
break;
case BLORP_HIZ_OP_DEPTH_CLEAR:
dw1 |= GEN8_WM_HZ_DEPTH_CLEAR;
/* The "Clear Rectangle X Max" (and Y Max) fields are exclusive,
* rather than inclusive, and limited to 16383. This means that
* for a 16384x16384 render target, we would miss the last row
* or column of pixels along the edge.
*
* To work around this, we have to set the "Full Surface Depth
* and Stencil Clear" bit. We can do this in all cases because
* we always clear the full rectangle anyway. We'll need to
* change this if we ever add scissored clear support.
*/
dw1 |= GEN8_WM_HZ_FULL_SURFACE_DEPTH_CLEAR;
break;
case BLORP_HIZ_OP_NONE:
unreachable("Should not get here.");
}
if (mt->num_samples > 0)
dw1 |= SET_FIELD(ffs(mt->num_samples) - 1, GEN8_WM_HZ_NUM_SAMPLES);
BEGIN_BATCH(5);
OUT_BATCH(_3DSTATE_WM_HZ_OP << 16 | (5 - 2));
OUT_BATCH(dw1);
OUT_BATCH(0);
OUT_BATCH(SET_FIELD(rect_width, GEN8_WM_HZ_CLEAR_RECTANGLE_X_MAX) |
SET_FIELD(rect_height, GEN8_WM_HZ_CLEAR_RECTANGLE_Y_MAX));
OUT_BATCH(SET_FIELD(0xFFFF, GEN8_WM_HZ_SAMPLE_MASK));
ADVANCE_BATCH();
/* Emit a PIPE_CONTROL with "Post-Sync Operation" set to "Write Immediate
* Data", and no other bits set. This causes 3DSTATE_WM_HZ_OP's state to
* take effect, and spawns a rectangle primitive.
*/
brw_emit_pipe_control_write(brw,
PIPE_CONTROL_WRITE_IMMEDIATE,
brw->workaround_bo, 0, 0, 0);
/* Emit 3DSTATE_WM_HZ_OP again to disable the state overrides. */
BEGIN_BATCH(5);
OUT_BATCH(_3DSTATE_WM_HZ_OP << 16 | (5 - 2));
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
OUT_BATCH(0);
ADVANCE_BATCH();
/*
* From the Broadwell PRM, volume 7, "Depth Buffer Clear":
*
* Depth buffer clear pass using any of the methods (WM_STATE, 3DSTATE_WM
* or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL command with
* DEPTH_STALL bit and Depth FLUSH bits "set" before starting to render.
* DepthStall and DepthFlush are not needed between consecutive depth
* clear passes nor is it required if th e depth clear pass was done with
* "full_surf_clear" bit set in the 3DSTATE_WM_HZ_OP.
*
* TODO: Such as the spec says, this could be conditional.
*/
brw_emit_pipe_control_flush(brw,
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
PIPE_CONTROL_DEPTH_STALL);
/* Mark this buffer as needing a TC flush, as we've rendered to it. */
brw_render_cache_set_add_bo(brw, mt->bo);
/* We've clobbered all of the depth packets, and the drawing rectangle,
* so we need to ensure those packets are re-emitted before the next
* primitive.
*
* Setting _NEW_DEPTH and _NEW_BUFFERS covers it, but is rather overkill.
*/
brw->NewGLState |= _NEW_DEPTH | _NEW_BUFFERS;
}