/*
* Copyright (c) 2016 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
*/
/** @file hsw_queryobj.c
*
* Support for query buffer objects (GL_ARB_query_buffer_object) on Haswell+.
*/
#include "main/imports.h"
#include "brw_context.h"
#include "brw_defines.h"
#include "intel_batchbuffer.h"
#include "intel_buffer_objects.h"
/*
* GPR0 = 80 * GPR0;
*/
static void
mult_gpr0_by_80(struct brw_context *brw)
{
static const uint32_t maths[] = {
MI_MATH_ALU2(LOAD, SRCA, R0),
MI_MATH_ALU2(LOAD, SRCB, R0),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STORE, R1, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R1),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STORE, R1, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R1),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STORE, R1, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R1),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(ADD),
/* GPR1 = 16 * GPR0 */
MI_MATH_ALU2(STORE, R1, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R1),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STORE, R2, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R2),
MI_MATH_ALU2(LOAD, SRCB, R2),
MI_MATH_ALU0(ADD),
/* GPR2 = 64 * GPR0 */
MI_MATH_ALU2(STORE, R2, ACCU),
MI_MATH_ALU2(LOAD, SRCA, R1),
MI_MATH_ALU2(LOAD, SRCB, R2),
MI_MATH_ALU0(ADD),
/* GPR0 = 80 * GPR0 */
MI_MATH_ALU2(STORE, R0, ACCU),
};
BEGIN_BATCH(1 + ARRAY_SIZE(maths));
OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
for (int m = 0; m < ARRAY_SIZE(maths); m++)
OUT_BATCH(maths[m]);
ADVANCE_BATCH();
}
/*
* GPR0 = GPR0 & ((1ull << n) - 1);
*/
static void
keep_gpr0_lower_n_bits(struct brw_context *brw, uint32_t n)
{
static const uint32_t maths[] = {
MI_MATH_ALU2(LOAD, SRCA, R0),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(AND),
MI_MATH_ALU2(STORE, R0, ACCU),
};
assert(n < 64);
brw_load_register_imm64(brw, HSW_CS_GPR(1), (1ull << n) - 1);
BEGIN_BATCH(1 + ARRAY_SIZE(maths));
OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
for (int m = 0; m < ARRAY_SIZE(maths); m++)
OUT_BATCH(maths[m]);
ADVANCE_BATCH();
}
/*
* GPR0 = GPR0 << 30;
*/
static void
shl_gpr0_by_30_bits(struct brw_context *brw)
{
/* First we mask 34 bits of GPR0 to prevent overflow */
keep_gpr0_lower_n_bits(brw, 34);
static const uint32_t shl_maths[] = {
MI_MATH_ALU2(LOAD, SRCA, R0),
MI_MATH_ALU2(LOAD, SRCB, R0),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STORE, R0, ACCU),
};
const uint32_t outer_count = 5;
const uint32_t inner_count = 6;
STATIC_ASSERT(outer_count * inner_count == 30);
const uint32_t cmd_len = 1 + inner_count * ARRAY_SIZE(shl_maths);
const uint32_t batch_len = cmd_len * outer_count;
BEGIN_BATCH(batch_len);
/* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
* 30 left shifts.
*/
for (int o = 0; o < outer_count; o++) {
/* Submit one MI_MATH to shift left by 6 bits */
OUT_BATCH(HSW_MI_MATH | (cmd_len - 2));
for (int i = 0; i < inner_count; i++)
for (int m = 0; m < ARRAY_SIZE(shl_maths); m++)
OUT_BATCH(shl_maths[m]);
}
ADVANCE_BATCH();
}
/*
* GPR0 = GPR0 >> 2;
*
* Note that the upper 30 bits of GPR0 are lost!
*/
static void
shr_gpr0_by_2_bits(struct brw_context *brw)
{
shl_gpr0_by_30_bits(brw);
brw_load_register_reg(brw, HSW_CS_GPR(0) + 4, HSW_CS_GPR(0));
brw_load_register_imm32(brw, HSW_CS_GPR(0) + 4, 0);
}
/*
* GPR0 = (GPR0 == 0) ? 0 : 1;
*/
static void
gpr0_to_bool(struct brw_context *brw)
{
static const uint32_t maths[] = {
MI_MATH_ALU2(LOAD, SRCA, R0),
MI_MATH_ALU1(LOAD0, SRCB),
MI_MATH_ALU0(ADD),
MI_MATH_ALU2(STOREINV, R0, ZF),
MI_MATH_ALU2(LOAD, SRCA, R0),
MI_MATH_ALU2(LOAD, SRCB, R1),
MI_MATH_ALU0(AND),
MI_MATH_ALU2(STORE, R0, ACCU),
};
brw_load_register_imm64(brw, HSW_CS_GPR(1), 1ull);
BEGIN_BATCH(1 + ARRAY_SIZE(maths));
OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
for (int m = 0; m < ARRAY_SIZE(maths); m++)
OUT_BATCH(maths[m]);
ADVANCE_BATCH();
}
static void
hsw_result_to_gpr0(struct gl_context *ctx, struct brw_query_object *query,
struct gl_buffer_object *buf, intptr_t offset,
GLenum pname, GLenum ptype)
{
struct brw_context *brw = brw_context(ctx);
assert(query->bo);
assert(pname != GL_QUERY_TARGET);
if (pname == GL_QUERY_RESULT_AVAILABLE) {
/* The query result availability is stored at offset 0 of the buffer. */
brw_load_register_mem64(brw,
HSW_CS_GPR(0),
query->bo,
I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION,
2 * sizeof(uint64_t));
return;
}
if (pname == GL_QUERY_RESULT) {
/* Since GL_QUERY_RESULT_NO_WAIT wasn't used, they want us to stall to
* make sure the query is available.
*/
brw_emit_pipe_control_flush(brw,
PIPE_CONTROL_CS_STALL |
PIPE_CONTROL_STALL_AT_SCOREBOARD);
}
if (query->Base.Target == GL_TIMESTAMP) {
brw_load_register_mem64(brw,
HSW_CS_GPR(0),
query->bo,
I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION,
0 * sizeof(uint64_t));
} else {
brw_load_register_mem64(brw,
HSW_CS_GPR(1),
query->bo,
I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION,
0 * sizeof(uint64_t));
brw_load_register_mem64(brw,
HSW_CS_GPR(2),
query->bo,
I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION,
1 * sizeof(uint64_t));
BEGIN_BATCH(5);
OUT_BATCH(HSW_MI_MATH | (5 - 2));
OUT_BATCH(MI_MATH_ALU2(LOAD, SRCA, R2));
OUT_BATCH(MI_MATH_ALU2(LOAD, SRCB, R1));
OUT_BATCH(MI_MATH_ALU0(SUB));
OUT_BATCH(MI_MATH_ALU2(STORE, R0, ACCU));
ADVANCE_BATCH();
}
switch (query->Base.Target) {
case GL_FRAGMENT_SHADER_INVOCATIONS_ARB:
/* Implement the "WaDividePSInvocationCountBy4:HSW,BDW" workaround:
* "Invocation counter is 4 times actual. WA: SW to divide HW reported
* PS Invocations value by 4."
*
* Prior to Haswell, invocation count was counted by the WM, and it
* buggily counted invocations in units of subspans (2x2 unit). To get the
* correct value, the CS multiplied this by 4. With HSW the logic moved,
* and correctly emitted the number of pixel shader invocations, but,
* whomever forgot to undo the multiply by 4.
*/
if (brw->gen == 8 || brw->is_haswell)
shr_gpr0_by_2_bits(brw);
break;
case GL_TIME_ELAPSED:
case GL_TIMESTAMP:
mult_gpr0_by_80(brw);
if (query->Base.Target == GL_TIMESTAMP) {
keep_gpr0_lower_n_bits(brw, 36);
}
break;
case GL_ANY_SAMPLES_PASSED:
case GL_ANY_SAMPLES_PASSED_CONSERVATIVE:
gpr0_to_bool(brw);
break;
}
}
/*
* Store immediate data into the user buffer using the requested size.
*/
static void
store_query_result_imm(struct brw_context *brw, drm_intel_bo *bo,
uint32_t offset, GLenum ptype, uint64_t imm)
{
switch (ptype) {
case GL_INT:
case GL_UNSIGNED_INT:
brw_store_data_imm32(brw, bo, offset, imm);
break;
case GL_INT64_ARB:
case GL_UNSIGNED_INT64_ARB:
brw_store_data_imm64(brw, bo, offset, imm);
break;
default:
unreachable("Unexpected result type");
}
}
static void
set_predicate(struct brw_context *brw, drm_intel_bo *query_bo)
{
brw_load_register_imm64(brw, MI_PREDICATE_SRC1, 0ull);
/* Load query availability into SRC0 */
brw_load_register_mem64(brw, MI_PREDICATE_SRC0, query_bo,
I915_GEM_DOMAIN_INSTRUCTION, 0,
2 * sizeof(uint64_t));
/* predicate = !(query_availability == 0); */
BEGIN_BATCH(1);
OUT_BATCH(GEN7_MI_PREDICATE |
MI_PREDICATE_LOADOP_LOADINV |
MI_PREDICATE_COMBINEOP_SET |
MI_PREDICATE_COMPAREOP_SRCS_EQUAL);
ADVANCE_BATCH();
}
/*
* Store data from the register into the user buffer using the requested size.
* The write also enables the predication to prevent writing the result if the
* query has not finished yet.
*/
static void
store_query_result_reg(struct brw_context *brw, drm_intel_bo *bo,
uint32_t offset, GLenum ptype, uint32_t reg,
const bool pipelined)
{
uint32_t cmd_size = brw->gen >= 8 ? 4 : 3;
uint32_t dwords = (ptype == GL_INT || ptype == GL_UNSIGNED_INT) ? 1 : 2;
assert(brw->gen >= 6);
BEGIN_BATCH(dwords * cmd_size);
for (int i = 0; i < dwords; i++) {
OUT_BATCH(MI_STORE_REGISTER_MEM |
(pipelined ? MI_STORE_REGISTER_MEM_PREDICATE : 0) |
(cmd_size - 2));
OUT_BATCH(reg + 4 * i);
if (brw->gen >= 8) {
OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
} else {
OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION,
I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
}
}
ADVANCE_BATCH();
}
static void
hsw_store_query_result(struct gl_context *ctx, struct gl_query_object *q,
struct gl_buffer_object *buf, intptr_t offset,
GLenum pname, GLenum ptype)
{
struct brw_context *brw = brw_context(ctx);
struct brw_query_object *query = (struct brw_query_object *)q;
struct intel_buffer_object *bo = intel_buffer_object(buf);
const bool pipelined = brw_is_query_pipelined(query);
if (pname == GL_QUERY_TARGET) {
store_query_result_imm(brw, bo->buffer, offset, ptype,
query->Base.Target);
return;
} else if (pname == GL_QUERY_RESULT_AVAILABLE && !pipelined) {
store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
} else if (query->bo) {
/* The query bo still around. Therefore, we:
*
* 1. Compute the current result in GPR0
* 2. Set the command streamer predicate based on query availability
* 3. (With predication) Write GPR0 to the requested buffer
*/
hsw_result_to_gpr0(ctx, query, buf, offset, pname, ptype);
if (pipelined)
set_predicate(brw, query->bo);
store_query_result_reg(brw, bo->buffer, offset, ptype, HSW_CS_GPR(0),
pipelined);
} else {
/* The query bo is gone, so the query must have been processed into
* client memory. In this case we can fill the buffer location with the
* requested data using MI_STORE_DATA_IMM.
*/
switch (pname) {
case GL_QUERY_RESULT_AVAILABLE:
store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
break;
case GL_QUERY_RESULT_NO_WAIT:
case GL_QUERY_RESULT:
store_query_result_imm(brw, bo->buffer, offset, ptype,
q->Result);
break;
default:
unreachable("Unexpected result type");
}
}
}
/* Initialize hsw+-specific query object functions. */
void hsw_init_queryobj_functions(struct dd_function_table *functions)
{
gen6_init_queryobj_functions(functions);
functions->StoreQueryResult = hsw_store_query_result;
}