/*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f: .fnstart
#define END(f) .fnend; .size f, .-f;
.eabi_attribute 25,1 @Tag_ABI_align8_preserved
.arm
/* Fixed-point precision after vertical pass -- 16 bit data minus 1 sign and 1
* integer (bicubic has a little overshoot). It would also be possible to add
* a temporary DC bias to eliminate the sign bit for more precision, but that's
* extra arithmetic.
*/
.set VERTBITS, 14
/* The size of the scratch buffer in which we store our vertically convolved
* intermediates.
*/
.set CHUNKSHIFT, 7
.set CHUNKSIZE, (1 << CHUNKSHIFT)
/* The number of components processed in a single iteration of the innermost
* loop.
*/
.set VECSHIFT, 3
.set VECSIZE, (1<<VECSHIFT)
/* Read four different lines (except at edges where addresses may be clamped,
* which is why we don't simply take base and stride registers), and multiply
* and accumulate them by the coefficients in d6[0..3], leaving the results in
* q12. This gives eight 16-bit results representing a horizontal line of 2-8
* input pixels (depending on number of components per pixel) to be fed into
* the horizontal scaling pass.
*
* Input coefficients are 16-bit unsigned fixed-point (although [0] and [3] are
* known to represent negative values and VMLS is used to implement this).
* Output is VERTBITS signed fixed-point, which must leave room for a little
* bit of overshoot beyond [0,1.0).
*/
.macro vert8, dstlo=d24, dsthi=d25
vld1.u8 d16, [r4]!
vld1.u8 d18, [r5]!
vld1.u8 d20, [r6]!
vld1.u8 d22, [r7]!
vmovl.u8 q8, d16
vmovl.u8 q9, d18
vmovl.u8 q10, d20
vmovl.u8 q11, d22
vmull.u16 q12, d18, d6[1]
vmull.u16 q13, d19, d6[1]
vmlsl.u16 q12, d16, d6[0]
vmlsl.u16 q13, d17, d6[0]
vmlal.u16 q12, d20, d6[2]
vmlal.u16 q13, d21, d6[2]
vmlsl.u16 q12, d22, d6[3]
vmlsl.u16 q13, d23, d6[3]
/* Shift by 8 (bits per pixel), plus 16 (the fixed-point multiplies),
* minus VERTBITS (the number of fraction bits we want to keep from
* here on).
*/
vqshrn.s32 \dstlo, q12, #8 + 16 - VERTBITS
vqshrn.s32 \dsthi, q13, #8 + 16 - VERTBITS
.endm
/* As above, but only four 16-bit results into d25.
*/
.macro vert4
vld1.u32 d16[0], [r4]!
vld1.u32 d18[0], [r5]!
vld1.u32 d20[0], [r6]!
vld1.u32 d22[0], [r7]!
vmovl.u8 q8, d16
vmovl.u8 q9, d18
vmovl.u8 q10, d20
vmovl.u8 q11, d22
vmull.u16 q12, d18, d6[1]
vmlsl.u16 q12, d16, d6[0]
vmlal.u16 q12, d20, d6[2]
vmlsl.u16 q12, d22, d6[3]
vqshrn.s32 d25, q12, #8 + 16 - VERTBITS
.endm
/* During horizontal resize having CHUNKSIZE input available means being able
* to produce a varying amount of output, depending on the phase of the data.
* This function calculates the minimum number of VECSIZE chunks extracted from
* a CHUNKSIZE window (r1), and the threshold value for when the count will be
* one higher than that (r0).
* These work out, conveniently, to be the quotient and remainder from:
* (CHUNKSIZE + xinc * VECSIZE - 1) / (xinc * VECSIZE)
*
* The two values can be packed together in a uint64_t for convenience; and
* they are, in fact, used this way as an arithmetic short-cut later on.
*/
/* uint64_t rsdIntrinsicResize_oscctl_K(uint32_t xinc); */
ENTRY(rsdIntrinsicResize_oscctl_K)
lsl r2, r0, #VECSHIFT
movw r0, #:lower16:(CHUNKSIZE << 16) - 1
movt r0, #:upper16:(CHUNKSIZE << 16) - 1
add r0, r0, r2
#if defined(ARCH_ARM_USE_UDIV)
udiv r1, r0, r2
mls r0, r1, r2, r0
#else
clz r3, r2
clz r1, r0
subs r3, r3, r1
movlt r3, #0
mov r1, #1
lsl r2, r2, r3
lsl r3, r1, r3
mov r1, #0
1: cmp r2, r0
addls r1, r3
subls r0, r2
lsrs r3, r3, #1
lsr r2, r2, #1
bne 1b
#endif
bx lr
END(rsdIntrinsicResize_oscctl_K)
/* Iterate to generate the uchar1, uchar2, and uchar4 versions of the code.
* For the most part the vertical pass (the outer loop) is the same for all
* versions. Exceptions are handled in-line with conditional assembly.
*/
.irp comp, 1, 2, 4
.if \comp == 1
.set COMPONENT_SHIFT, 0
.elseif \comp == 2
.set COMPONENT_SHIFT, 1
.elseif \comp == 4
.set COMPONENT_SHIFT, 2
.else
.error "Unknown component count"
.endif
.set COMPONENT_COUNT, (1 << COMPONENT_SHIFT)
.set LOOP_OUTPUT_SIZE, (VECSIZE * COMPONENT_COUNT)
.set BUFFER_SIZE, (CHUNKSIZE * 2 + 4) * COMPONENT_COUNT * 2
.set OSC_STORE, (BUFFER_SIZE + 0)
.set OSCSTEP_STORE, (BUFFER_SIZE + 4)
.set OSCCTL_STORE, (BUFFER_SIZE + 8)
.set AVAIL_STORE, (BUFFER_SIZE + 16)
.set SP_STORE, (BUFFER_SIZE + 24) /* should be +20, but rounded up to make a legal constant somewhere */
/* void rsdIntrinsicResizeB\comp\()_K(
* uint8_t * restrict dst, // r0
* size_t count, // r1
* uint32_t xf, // r2
* uint32_t xinc, // r3
* uint8_t const * restrict srcn, // [sp] -> [sp,#104] -> r4
* uint8_t const * restrict src0, // [sp,#4] -> [sp,#108] -> r5
* uint8_t const * restrict src1, // [sp,#8] -> [sp,#112] -> r6
* uint8_t const * restrict src2, // [sp,#12] -> [sp,#116] -> r7
* size_t xclip, // [sp,#16] -> [sp,#120]
* size_t avail, // [sp,#20] -> [sp,#124] -> lr
* uint64_t osc_ctl, // [sp,#24] -> [sp,#128]
* int32_t const *yr); // [sp,#32] -> [sp,#136] -> d8 (copied to d6 for scalar access)
*/
ENTRY(rsdIntrinsicResizeB\comp\()_K)
push {r4,r5,r6,r7,r8,r9,r10,r11,r12,lr}
vpush {d8-d15}
/* align the working buffer on the stack to make it easy to use bit
* twiddling for address calculations and bounds tests.
*/
sub r12, sp, #BUFFER_SIZE + 32
mov lr, sp
bfc r12, #0, #CHUNKSHIFT + 1 + COMPONENT_SHIFT + 1
mov sp, r12
str lr, [sp,#SP_STORE]
ldr r8, [lr,#136] // yr
adr r9, 8f
vld1.s32 {q4}, [r8]
vld1.s16 {q5}, [r9]
vqmovun.s32 d8, q4 // yr
vdup.s16 q6, r2
vdup.s16 q7, r3
vmla.s16 q6, q5, q7 // vxf
vshl.s16 q7, q7, #VECSHIFT // vxinc
ldrd r4,r5, [lr,#104] // srcn, src0
ldrd r6,r7, [lr,#112] // src1, src2
/* Compute starting condition for oscillator used to compute ahead
* of time how many iterations are possible before needing to
* refill the working buffer. This is based on the fixed-point
* index of the last element in the vector of pixels processed in
* each iteration, counting up until it would overflow.
*/
sub r8, r2, r3
mov r9, r3, LSL #VECSHIFT
add r8, r8, r9
ldrd r10,r11, [lr,#128] // osc_ctl
str r8, [sp,#OSC_STORE]
str r9, [sp,#OSCSTEP_STORE]
str r10, [sp,#OSCCTL_STORE]
str r11, [sp,#OSCCTL_STORE+4]
ldrd r10,r11, [lr,#120] // xclip,avail
/* r4-r7 contain pointers to the four lines of input to be
* convolved. These pointers have been clamped vertically and
* horizontally (which is why it's not a simple row/stride pair),
* and the xclip argument (now in r10) indicates how many pixels
* from true the x position of the pointer is. This value should
* be 0, 1, or 2 only.
*
* Start by placing four pixels worth of input at the far end of
* the buffer. As many as two of these may be clipped, so four
* pixels are fetched, and then the first pixel is duplicated and
* the data shifted according to xclip. The source pointers are
* then also adjusted according to xclip so that subsequent fetches
* match.
*/
vmov d6, d8 /* make y coeffs available for vert4 and vert8 macros */
sub r8, r12, r10, LSL #COMPONENT_SHIFT + 1
add r9, r12, #(2 * CHUNKSIZE - 4) * COMPONENT_COUNT * 2
add r8, r8, #4 * COMPONENT_COUNT * 2
.if \comp == 1
vert4
vdup.s16 d24, d25[0]
vst1.s16 {q12}, [r12]
vld1.s16 {d24}, [r8]
vst1.s16 {d24}, [r9]
.elseif \comp == 2
vert8
vdup.u32 q11, d24[0]
vst1.s16 {q11,q12}, [r12]
vld1.s16 {q12}, [r8]
vst1.s16 {q12}, [r9]
.elseif \comp == 4
vert8 d28, d29
vert8 d30, d31
vmov.u64 d24, d28
vmov.u64 d25, d28
vmov.u64 d26, d28
vmov.u64 d27, d28
vst1.s16 {q12,q13}, [r12]!
vst1.s16 {q14,q15}, [r12]
sub r12, r12, #32
vld1.s16 {q11,q12}, [r8]
vst1.s16 {q11,q12}, [r9]
.endif
/* Count off four pixels into the working buffer, and move count to
* its new home.
*/
sub lr, r11, #4
/* Incoming pointers were to the first _legal_ pixel. Four pixels
* were read unconditionally, but some may have been discarded by
* xclip, so we rewind the pointers to compensate.
*/
sub r4, r4, r10, LSL #COMPONENT_SHIFT
sub r5, r5, r10, LSL #COMPONENT_SHIFT
sub r6, r6, r10, LSL #COMPONENT_SHIFT
sub r7, r7, r10, LSL #COMPONENT_SHIFT
/* First tap starts where we just pre-filled, at the end of the
* buffer.
*/
add r2, r2, #(CHUNKSIZE * 2 - 4) << 16
/* Use overflowing arithmetic to implement wraparound array
* indexing.
*/
mov r2, r2, LSL #(15 - CHUNKSHIFT)
mov r3, r3, LSL #(15 - CHUNKSHIFT)
str lr, [sp,#AVAIL_STORE]
/* Start of outermost loop.
* Fetch CHUNKSIZE pixels into scratch buffer, then calculate the
* number of iterations of the inner loop that can be performed and
* get into that.
*
* The fill is complicated by the possibility of running out of
* input before the scratch buffer is filled. If this isn't a risk
* then it's handled by the simple loop at 2:, otherwise the
* horrible loop at 3:.
*/
1: ldr lr, [sp,#AVAIL_STORE] /* get number of pixels available */
vmov d6, d8 /* put y scaling coefficients somewhere handy */
subs lr, #CHUNKSIZE
bge 2f /* if at least CHUNKSIZE are available... */
add lr, #CHUNKSIZE /* if they're not... */
b 4f
/* ..just sneaking a literal in here after this unconditional branch.. */
8: .hword 0, 1, 2, 3, 4, 5, 6, 7
/* basic fill loop, processing 8 bytes at a time until there are
* fewer than eight bytes available.
*/
3: vert8
sub lr, lr, #8 / COMPONENT_COUNT
vst1.s16 {q12}, [r12]!
4: cmp lr, #8 / COMPONENT_COUNT - 1
bgt 3b
.if \comp == 4
blt 3f
/* The last pixel (four bytes) if necessary */
vert4
.else
cmp lr, #1
blt 3f
/* The last pixels if necessary */
sub r4, r4, #8
sub r5, r5, #8
sub r6, r6, #8
sub r7, r7, #8
add r4, r4, lr, LSL #COMPONENT_SHIFT
add r5, r5, lr, LSL #COMPONENT_SHIFT
add r6, r6, lr, LSL #COMPONENT_SHIFT
add r7, r7, lr, LSL #COMPONENT_SHIFT
vert8
sub lr, sp, lr, LSL #COMPONENT_SHIFT + 1
sub sp, sp, #32
sub lr, lr, #16
.if \comp == 1
vdup.s16 q13, d25[3]
.elseif \comp == 2
vdup.u32 q13, d25[1]
.endif
vst1.s16 {q12,q13}, [sp]
vld1.s16 {q12}, [lr]
add sp, sp, #32
b 4f
.endif
/* Keep filling until we get to the end of this chunk of the buffer */
3:
.if \comp == 1
vdup.s16 q12, d25[3]
.elseif \comp == 2
vdup.u32 q12, d25[1]
.elseif \comp == 4
vmov.u64 d24, d25
.endif
4: vst1.s16 {q12}, [r12]!
tst r12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
bne 3b
b 4f
.align 4
2: /* Quickly pull a chunk of data into the working buffer.
*/
vert8
vst1.s16 {q12}, [r12]!
vert8
vst1.s16 {q12}, [r12]!
tst r12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
bne 2b
cmp lr, #0
bne 3f
4: /* if we end with 0 pixels left we'll have nothing handy to spread
* across to the right, so we rewind a bit.
*/
mov lr, #1
sub r4, r4, #COMPONENT_COUNT
sub r5, r5, #COMPONENT_COUNT
sub r6, r6, #COMPONENT_COUNT
sub r7, r7, #COMPONENT_COUNT
3: str lr, [sp,#AVAIL_STORE] /* done with available pixel count */
add lr, sp, #OSC_STORE
ldrd r8,r9, [lr,#0] /* need osc, osc_step soon */
ldrd r10,r11, [lr,#OSCCTL_STORE-OSC_STORE] /* need osc_ctl too */
/* copy four taps (width of cubic window) to far end for overflow
* address handling
*/
sub lr, r12, #CHUNKSIZE * COMPONENT_COUNT * 2
eor r12, lr, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
vld1.s16 {d28}, [lr]
.elseif \comp == 2
vld1.s16 {q14}, [lr]
.elseif \comp == 4
vld1.s16 {q14,q15}, [lr]
.endif
add lr, r12, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
vst1.s16 {d28}, [lr]
.elseif \comp == 2
vst1.s16 {q14}, [lr]
.elseif \comp == 4
vst1.s16 {q14,q15}, [lr]
.endif
/* r11 contains the maximum possible iteration count, but if r8 is
* greater than r10 then this indicates that the count must be
* reduced by one for this iteration to avoid reading past the end
* of the available data.
*/
cmp r10, r8
sbc lr, r11, #0
mla r8, lr, r9, r8
sub r8, r8, #(CHUNKSIZE << 16)
str r8, [sp,#OSC_STORE] /* done with osc */
/* prefer to count pixels, rather than vectors, to clarify the tail
* store case on exit.
*/
mov lr, lr, LSL #VECSHIFT
cmp lr, r1
movgt lr, r1
sub r1, r1, lr
mov lr, lr, LSL #COMPONENT_SHIFT
vmov.i16 d10, #3
vmov.i16 d11, #0x8000
cmp lr, #0
bgt 3f
cmp r1, #0
bgt 1b /* an extreme case where we shouldn't use code in this structure */
b 9f
.align 4
2: /* Inner loop continues here, but starts at 3:, see end of loop
* below for explanation. */
.if LOOP_OUTPUT_SIZE == 4
vst1.u32 {d16[0]}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 8
vst1.u8 {d16}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 16
vst1.u8 {q8}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 32
vst1.u8 {q8,q9}, [r0]!
.endif
/* Inner loop: here the four x coefficients for each tap are
* calculated in vector code, and the addresses are calculated in
* scalar code, and these calculations are interleaved.
*/
3: vshr.u16 q8, q6, #1
mov r8, r2, LSR #(31 - CHUNKSHIFT)
vqrdmulh.s16 q9, q8, q8
add r2, r2, r3
vqrdmulh.s16 q10, q9, q8
mov r9, r2, LSR #(31 - CHUNKSHIFT)
vshll.s16 q11, d18, #2
vshll.s16 q12, d19, #2
add r2, r2, r3
vmlsl.s16 q11, d20, d10
vmlsl.s16 q12, d21, d10
mov r10, r2, LSR #(31 - CHUNKSHIFT)
vhadd.s16 q0, q10, q8
add r2, r2, r3
vsub.s16 q0, q9, q0
mov r11, r2, LSR #(31 - CHUNKSHIFT)
vaddw.s16 q1, q11, d18
vaddw.s16 q13, q12, d19
add r2, r2, r3
vshrn.s32 d2, q1, #1
vshrn.s32 d3, q13, #1
add r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
vsub.s16 d2, d2, d11
vsub.s16 d3, d3, d11 // TODO: find a wider d11 and use q-reg operation
add r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
vaddw.s16 q2, q11, d16
vaddw.s16 q13, q12, d17
add r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
vshrn.s32 d4, q2, #1
vshrn.s32 d5, q13, #1
add r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
vneg.s16 q2, q2
vhsub.s16 q3, q10, q9
/* increment the x fractional parts (oveflow is ignored, as the
* scalar arithmetic shadows this addition with full precision).
*/
vadd.s16 q6, q6, q7
/* At this point we have four pointers in r8-r11, pointing to the
* four taps in the scratch buffer that must be convolved together
* to produce an output pixel (one output pixel per pointer).
* These pointers usually overlap, but their spacing is irregular
* so resolving the redundancy through L1 is a pragmatic solution.
*
* The scratch buffer is made of signed 16-bit data, holding over
* some extra precision, and overshoot, from the vertical pass.
*
* We also have the 16-bit unsigned fixed-point weights for each
* of the four taps in q0 - q3. That's eight pixels worth of
* coefficients when we have only four pointers, so calculations
* for four more pixels are interleaved with the fetch and permute
* code for each variant in the following code.
*
* The data arrangement is less than ideal for any pixel format,
* but permuting loads help to mitigate most of the problems.
*
* Note also that the two outside taps of a bicubic are negative,
* but these coefficients are unsigned. The sign is hard-coded by
* use of multiply-and-subtract operations.
*/
.if \comp == 1
/* The uchar 1 case.
* Issue one lanewise vld4.s16 to load four consecutive pixels from
* one pointer (one pixel) into four different registers; then load
* four consecutive s16 values from the next pointer (pixel) into
* the next lane of those four registers, etc., so that we finish
* with q12 - q15 representing the four taps, and each lane
* representing a separate pixel.
*
* The first vld4 uses a splat to avoid any false dependency on
* the previous state of the register.
*/
vld4.s16 {d24[],d26[],d28[],d30[]}, [r8]
mov r8, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.s16 {d24[1],d26[1],d28[1],d30[1]}, [r9]
add r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
mov r9, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.s16 {d24[2],d26[2],d28[2],d30[2]}, [r10]
add r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
mov r10, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.s16 {d24[3],d26[3],d28[3],d30[3]}, [r11]
add r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
mov r11, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.s16 {d25[],d27[],d29[],d31[]}, [r8]
add r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
vld4.s16 {d25[1],d27[1],d29[1],d31[1]}, [r9]
vld4.s16 {d25[2],d27[2],d29[2],d31[2]}, [r10]
vld4.s16 {d25[3],d27[3],d29[3],d31[3]}, [r11]
vmull.s16 q8, d24, d0
vmull.s16 q9, d25, d1
vmlsl.s16 q8, d26, d2
vmlsl.s16 q9, d27, d3
vmlsl.s16 q8, d28, d4
vmlsl.s16 q9, d29, d5
vmlal.s16 q8, d30, d6
vmlal.s16 q9, d31, d7
subs lr, lr, #LOOP_OUTPUT_SIZE
vqrshrn.s32 d16, q8, #15
vqrshrn.s32 d17, q9, #15
vqrshrun.s16 d16, q8, #VERTBITS - 8
.elseif \comp == 2
/* The uchar2 case:
* This time load pairs of values into adjacent lanes in q12 - q15
* by aliasing them as u32 data; leaving room for only four pixels,
* so the process has to be done twice. This also means that the
* coefficient registers fail to align with the coefficient data
* (eight separate pixels), so that has to be doubled-up to match.
*/
vld4.u32 {d24[],d26[],d28[],d30[]}, [r8]
mov r8, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.u32 {d24[1],d26[1],d28[1],d30[1]}, [r9]
add r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
mov r9, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.u32 {d25[],d27[],d29[],d31[]}, [r10]
add r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
mov r10, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld4.u32 {d25[1],d27[1],d29[1],d31[1]}, [r11]
add r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
mov r11, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
/* double-up coefficients to align with component pairs */
vmov d20, d0
add r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
vmov d21, d2
vmov d22, d4
vmov d23, d6
vzip.s16 d0, d20
vzip.s16 d2, d21
vzip.s16 d4, d22
vzip.s16 d6, d23
vmull.s16 q8, d24, d0
vmull.s16 q9, d25, d20
vmlsl.s16 q8, d26, d2
vmlsl.s16 q9, d27, d21
vmlsl.s16 q8, d28, d4
vmlsl.s16 q9, d29, d22
vmlal.s16 q8, d30, d6
vmlal.s16 q9, d31, d23
vqrshrn.s32 d16, q8, #15
vqrshrn.s32 d17, q9, #15
vld4.u32 {d24[],d26[],d28[],d30[]}, [r8]
vld4.u32 {d24[1],d26[1],d28[1],d30[1]}, [r9]
vld4.u32 {d25[],d27[],d29[],d31[]}, [r10]
vld4.u32 {d25[1],d27[1],d29[1],d31[1]}, [r11]
/* double-up coefficients to align with component pairs */
vmov d0, d1
vmov d2, d3
vmov d4, d5
vmov d6, d7
vzip.s16 d0, d1
vzip.s16 d2, d3
vzip.s16 d4, d5
vzip.s16 d6, d7
vmull.s16 q10, d24, d0
vmull.s16 q11, d25, d1
vmlsl.s16 q10, d26, d2
vmlsl.s16 q11, d27, d3
vmlsl.s16 q10, d28, d4
vmlsl.s16 q11, d29, d5
vmlal.s16 q10, d30, d6
vmlal.s16 q11, d31, d7
subs lr, lr, #LOOP_OUTPUT_SIZE
vqrshrn.s32 d18, q10, #15
vqrshrn.s32 d19, q11, #15
vqrshrun.s16 d16, q8, #VERTBITS - 8
vqrshrun.s16 d17, q9, #VERTBITS - 8
.elseif \comp == 4
/* The uchar4 case.
* This case is comparatively painless because four s16s are the
* smallest addressable unit for a vmul-by-scalar. Rather than
* permute the data, simply arrange the multiplies to suit the way
* the data comes in. That's a lot of data, though, so things
* progress in pairs of pixels at a time.
*/
vld1.s16 {q12,q13}, [r8]
mov r8, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld1.s16 {q14,q15}, [r9]
add r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
mov r9, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vmull.s16 q8, d24, d0[0]
vmull.s16 q9, d28, d0[1]
vmlsl.s16 q8, d25, d2[0]
vmlsl.s16 q9, d29, d2[1]
vmlsl.s16 q8, d26, d4[0]
vmlsl.s16 q9, d30, d4[1]
vmlal.s16 q8, d27, d6[0]
vmlal.s16 q9, d31, d6[1]
/* And two more... */
vld1.s16 {q12,q13}, [r10]
add r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
mov r10, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vld1.s16 {q14,q15}, [r11]
add r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
mov r11, r2, LSR #(31 - CHUNKSHIFT)
add r2, r2, r3
vqrshrn.s32 d16, q8, #15
add r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
vqrshrn.s32 d17, q9, #15
vmull.s16 q10, d24, d0[2]
vmull.s16 q11, d28, d0[3]
vmlsl.s16 q10, d25, d2[2]
vmlsl.s16 q11, d29, d2[3]
vmlsl.s16 q10, d26, d4[2]
vmlsl.s16 q11, d30, d4[3]
vmlal.s16 q10, d27, d6[2]
vmlal.s16 q11, d31, d6[3]
vqrshrn.s32 d18, q10, #15
vqrshrn.s32 d19, q11, #15
vqrshrun.s16 d16, q8, #VERTBITS - 8
vqrshrun.s16 d17, q9, #VERTBITS - 8
/* And two more... */
vld1.s16 {q12,q13}, [r8]
vld1.s16 {q14,q15}, [r9]
vmull.s16 q10, d24, d1[0]
vmull.s16 q11, d28, d1[1]
vmlsl.s16 q10, d25, d3[0]
vmlsl.s16 q11, d29, d3[1]
vmlsl.s16 q10, d26, d5[0]
vmlsl.s16 q11, d30, d5[1]
vmlal.s16 q10, d27, d7[0]
vmlal.s16 q11, d31, d7[1]
/* And two more... */
vld1.s16 {q12,q13}, [r10]
vld1.s16 {q14,q15}, [r11]
subs lr, lr, #LOOP_OUTPUT_SIZE
vqrshrn.s32 d18, q10, #15
vqrshrn.s32 d19, q11, #15
vmull.s16 q10, d24, d1[2]
vmull.s16 q11, d28, d1[3]
vmlsl.s16 q10, d25, d3[2]
vmlsl.s16 q11, d29, d3[3]
vmlsl.s16 q10, d26, d5[2]
vmlsl.s16 q11, d30, d5[3]
vmlal.s16 q10, d27, d7[2]
vmlal.s16 q11, d31, d7[3]
vqrshrn.s32 d20, q10, #15
vqrshrn.s32 d21, q11, #15
vqrshrun.s16 d18, q9, #VERTBITS - 8
vqrshrun.s16 d19, q10, #VERTBITS - 8
.endif
bgt 2b /* continue inner loop */
/* The inner loop has already been limited to ensure that none of
* the earlier iterations could overfill the output, so the store
* appears within the loop but after the conditional branch (at the
* top). At the end, provided it won't overfill, perform the final
* store here. If it would, then break out to the tricky tail case
* instead.
*/
blt 1f
/* Store the amount of data appropriate to the configuration of the
* instance being assembled.
*/
.if LOOP_OUTPUT_SIZE == 4
vst1.u32 {d16[0]}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 8
vst1.u8 {d16}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 16
vst1.u8 {q8}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 32
vst1.u8 {q8,q9}, [r0]!
.endif
b 1b /* resume outer loop */
/* Partial tail store case:
* Different versions of the code need different subsets of the
* following partial stores. Here the number of components and the
* size of the chunk of data produced by each inner loop iteration
* is tested to figure out whether or not each phrase is relevant.
*/
.if 16 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 16
1: tst lr, #16
beq 1f
vst1.u8 {q8}, [r0]!
vmov q8, q9
.endif
.if 8 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 8
1: tst lr, #8
beq 1f
vst1.u8 {d16}, [r0]!
vmov.u8 d16, d17
.endif
.if 4 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 4
1: tst lr, #4
beq 1f
vst1.u32 {d16[0]}, [r0]!
vext.u32 d16, d16, d16, #1
.endif
.if 2 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 2
1: tst lr, #2
beq 1f
vst1.u16 {d16[0]}, [r0]!
vext.u16 d16, d16, d16, #1
.endif
.if 1 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 1
1: tst lr, #1
beq 1f
vst1.u8 {d16[0]}, [r0]!
.endif
1:
9: ldr sp, [sp,#SP_STORE]
vpop {d8-d15}
pop {r4,r5,r6,r7,r8,r9,r10,r11,r12,pc}
END(rsdIntrinsicResizeB\comp\()_K)
.endr