/* * Copyright (c) 2017 Imagination Technologies. * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with * the distribution. * * Neither the name of Imagination Technologies nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include <string.h> #include <stdint.h> #define ENABLE_PREFETCH 1 #define STRNG(X) #X #define PREFETCH(src_ptr, offset) \ asm("pref 0, " STRNG(offset) "(%[src]) \n\t" : : [src] "r" (src_ptr)); #if !defined(UNALIGNED_INSTR_SUPPORT) /* does target have unaligned lw/ld/ualw/uald instructions? */ #define UNALIGNED_INSTR_SUPPORT 0 #if __mips_isa_rev < 6 && !__mips1 #undef UNALIGNED_INSTR_SUPPORT #define UNALIGNED_INSTR_SUPPORT 1 #endif #endif #if !defined(HW_UNALIGNED_SUPPORT) /* Does target have hardware support for unaligned accesses? */ #define HW_UNALIGNED_SUPPORT 0 #if __mips_isa_rev >= 6 #undef HW_UNALIGNED_SUPPORT #define HW_UNALIGNED_SUPPORT 1 #endif #endif #define SIZEOF_reg_t 4 #if _MIPS_SIM == _ABIO32 typedef unsigned long reg_t; typedef struct bits { reg_t B0:8, B1:8, B2:8, B3:8; } bits_t; #else #undef SIZEOF_reg_t #define SIZEOF_reg_t 8 typedef unsigned long long reg_t; typedef struct bits { reg_t B0:8, B1:8, B2:8, B3:8, B4:8, B5:8, B6:8, B7:8; } bits_t; #endif /* This union assumes that small structures can be in registers. If not, then memory accesses will be done - not optimal, but ok. */ typedef union { reg_t v; bits_t b; } bitfields_t; #define do_bitfield(__i) \ if (x.b.B##__i != y.b.B##__i) return x.b.B##__i - y.b.B##__i; /* pull apart the words to find the first differing unsigned byte. */ static int __attribute__ ((noinline)) do_by_bitfields (reg_t a, reg_t b) { bitfields_t x, y; x.v = a; y.v = b; do_bitfield (0); do_bitfield (1); do_bitfield (2); #if SIZEOF_reg_t == 4 return x.b.B3 - y.b.B3; #else do_bitfield (3); do_bitfield (4); do_bitfield (5); do_bitfield (6); return x.b.B7 - y.b.B7; #endif } /* This code is called when aligning a pointer, there are remaining bytes after doing word compares, or architecture does not have some form of unaligned support. */ static inline int __attribute__ ((always_inline)) do_bytes (const void *a, const void *b, unsigned long len) { unsigned char *x = (unsigned char *) a; unsigned char *y = (unsigned char *) b; unsigned long i; /* 'len' might be zero here, so preloading the first two values before the loop may access unallocated memory. */ for (i = 0; i < len; i++) { if (*x != *y) return *x - *y; x++; y++; } return 0; } #if !HW_UNALIGNED_SUPPORT #if UNALIGNED_INSTR_SUPPORT /* for MIPS GCC, there are no unaligned builtins - so this struct forces the compiler to treat the pointer access as unaligned. */ struct ulw { reg_t uli; } __attribute__ ((packed)); /* first pointer is not aligned while second pointer is. */ static int unaligned_words (const struct ulw *a, const reg_t *b, unsigned long words, unsigned long bytes) { #if ENABLE_PREFETCH /* prefetch pointer aligned to 32 byte boundary */ const reg_t *pref_ptr = (const reg_t *) (((uintptr_t) b + 31) & ~31); const reg_t *pref_ptr_a = (const reg_t *) (((uintptr_t) a + 31) & ~31); #endif for (; words >= 16; words -= 8) { #if ENABLE_PREFETCH pref_ptr += 8; PREFETCH(pref_ptr, 0); PREFETCH(pref_ptr, 32); pref_ptr_a += 8; PREFETCH(pref_ptr_a, 0); PREFETCH(pref_ptr_a, 32); #endif reg_t x0 = a[0].uli, x1 = a[1].uli; reg_t x2 = a[2].uli, x3 = a[3].uli; reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); x0 = a[4].uli; x1 = a[5].uli; x2 = a[6].uli; x3 = a[7].uli; y0 = b[4]; y1 = b[5]; y2 = b[6]; y3 = b[7]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); a += 8; b += 8; } for (; words >= 4; words -= 4) { reg_t x0 = a[0].uli, x1 = a[1].uli; reg_t x2 = a[2].uli, x3 = a[3].uli; reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); a += 4; b += 4; } /* do remaining words. */ while (words--) { reg_t x0 = a->uli; reg_t y0 = *b; a += 1; b += 1; if (x0 != y0) return do_by_bitfields (x0, y0); } /* mop up any remaining bytes. */ return do_bytes (a, b, bytes); } #else /* no HW support or unaligned lw/ld/ualw/uald instructions. */ static int unaligned_words (const reg_t *a, const reg_t *b, unsigned long words, unsigned long bytes) { return do_bytes (a, b, (sizeof (reg_t) * words) + bytes); } #endif /* UNALIGNED_INSTR_SUPPORT */ #endif /* HW_UNALIGNED_SUPPORT */ /* both pointers are aligned, or first isn't and HW support for unaligned. */ static int aligned_words (const reg_t *a, const reg_t *b, unsigned long words, unsigned long bytes) { #if ENABLE_PREFETCH /* prefetch pointer aligned to 32 byte boundary */ const reg_t *pref_ptr = (const reg_t *) (((uintptr_t) b + 31) & ~31); const reg_t *pref_ptr_a = (const reg_t *) (((uintptr_t) a + 31) & ~31); #endif for (; words >= 24; words -= 12) { #if ENABLE_PREFETCH pref_ptr += 12; PREFETCH(pref_ptr, 0); PREFETCH(pref_ptr, 32); PREFETCH(pref_ptr, 64); pref_ptr_a += 12; PREFETCH(pref_ptr_a, 0); PREFETCH(pref_ptr_a, 32); PREFETCH(pref_ptr_a, 64); #endif reg_t x0 = a[0], x1 = a[1], x2 = a[2], x3 = a[3]; reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); x0 = a[4]; x1 = a[5]; x2 = a[6]; x3 = a[7]; y0 = b[4]; y1 = b[5]; y2 = b[6]; y3 = b[7]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); x0 = a[8]; x1 = a[9]; x2 = a[10]; x3 = a[11]; y0 = b[8]; y1 = b[9]; y2 = b[10]; y3 = b[11]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); a += 12; b += 12; } for (; words >= 4; words -= 4) { reg_t x0 = a[0], x1 = a[1], x2 = a[2], x3 = a[3]; reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3]; if (x0 != y0) return do_by_bitfields (x0, y0); if (x1 != y1) return do_by_bitfields (x1, y1); if (x2 != y2) return do_by_bitfields (x2, y2); if (x3 != y3) return do_by_bitfields (x3, y3); a += 4; b += 4; } /* do remaining words. */ while (words--) { reg_t x0 = *a; reg_t y0 = *b; a += 1; b += 1; if (x0 != y0) return do_by_bitfields (x0, y0); } /* mop up any remaining bytes. */ return do_bytes (a, b, bytes); } int memcmp (const void *a, const void *b, size_t len) { unsigned long bytes, words; /* shouldn't hit that often. */ if (len < sizeof (reg_t) * 4) { return do_bytes (a, b, len); } /* Align the second pointer to word/dword alignment. Note that the pointer is only 32-bits for o32/n32 ABIs. For n32, loads are done as 64-bit while address remains 32-bit. */ bytes = ((unsigned long) b) % sizeof (reg_t); if (bytes) { int res; bytes = sizeof (reg_t) - bytes; if (bytes > len) bytes = len; res = do_bytes (a, b, bytes); if (res || len == bytes) return res; len -= bytes; a = (const void *) (((unsigned char *) a) + bytes); b = (const void *) (((unsigned char *) b) + bytes); } /* Second pointer now aligned. */ words = len / sizeof (reg_t); bytes = len % sizeof (reg_t); #if HW_UNALIGNED_SUPPORT /* treat possible unaligned first pointer as aligned. */ return aligned_words (a, b, words, bytes); #else if (((unsigned long) a) % sizeof (reg_t) == 0) { return aligned_words (a, b, words, bytes); } /* need to use unaligned instructions on first pointer. */ return unaligned_words (a, b, words, bytes); #endif }