From d803198983d6f680e3590aab5ce26805a8297545 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:13:31 +0200 Subject: [PATCH 01/15] lib/libc/aarch64/string: add strcmp SIMD implementation This changeset includes a port of the SIMD implementation of strcmp for amd64 to Aarch64. Below is a description of its method as described in D41971. The basic idea is to process the bulk of the string in aligned blocks of 16 bytes such that one string runs ahead and the other runs behind. The string that runs ahead is checked for NUL bytes, the one that runs behind is compared with the corresponding chunk of the string that runs ahead. This trades an extra load per iteration for the very complicated block-reassembly needed in the other implementations (bionic, glibc). On the flip side, we need two code paths depending on the relative alignment of the two buffers. The initial part of the string is compared directly if it is known not to cross a page boundary. Otherwise, a complex slow path to avoid crossing into unmapped memory commences. Performance is better in most cases than the existing implementation from the Arm Optimized Routines repository. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D45839 --- lib/libc/aarch64/string/Makefile.inc | 4 +- lib/libc/aarch64/string/strcmp.S | 350 +++++++++++++++++++++++++++ 2 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strcmp.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index cabc79e4f351..bb74fec75e4b 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -13,13 +13,15 @@ AARCH64_STRING_FUNCS= \ stpcpy \ strchr \ strchrnul \ - strcmp \ strcpy \ strlen \ strncmp \ strnlen \ strrchr + +MDSRCS+= \ + strcmp.S # # Add the above functions. Generate an asm file that includes the needed # Arm Optimized Routines file defining the function name to the libc name. diff --git a/lib/libc/aarch64/string/strcmp.S b/lib/libc/aarch64/string/strcmp.S new file mode 100644 index 000000000000..e8418dfc6763 --- /dev/null +++ b/lib/libc/aarch64/string/strcmp.S @@ -0,0 +1,350 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include +#include + + .weak strcmp + .set strcmp, __strcmp + .text + +ENTRY(__strcmp) + + bic x8, x0, #0xf // x0 aligned to the boundary + and x9, x0, #0xf // x9 is the offset + bic x10, x1, #0xf // x1 aligned to the boundary + and x11, x1, #0xf // x11 is the offset + + mov x13, #-1 + + /* + * Check if either string is located at end of page to avoid crossing + * into unmapped page. If so, we load 16 bytes from the nearest + * alignment boundary and shift based on the offset. + */ + + add x3, x0, #16 // end of head + add x4, x1, #16 + eor x3, x3, x0 + eor x4, x4, x1 // bits that changed + orr x3, x3, x4 // in either str1 or str2 + tbz w3, #PAGE_SHIFT, .Lbegin + + ldr q0, [x8] // load aligned head + ldr q2, [x10] + + lsl x14, x9, #2 + lsl x15, x11, #2 + lsl x3, x13, x14 // string head + lsl x4, x13, x15 + + cmeq v5.16b, v0.16b, #0 + cmeq v6.16b, v2.16b, #0 + + shrn v5.8b, v5.8h, #4 + shrn v6.8b, v6.8h, #4 + fmov x5, d5 + fmov x6, d6 + + adrp x2, shift_data + add x2, x2, :lo12:shift_data + + /* heads may cross page boundary, avoid unmapped loads */ + tst x5, x3 + b.eq 0f + + ldr q4, [x2, x9] // load permutation table + tbl v0.16b, {v0.16b}, v4.16b + + b 1f + .p2align 4 +0: + ldr q0, [x0] // load true head +1: + tst x6, x4 + b.eq 0f + + ldr q4, [x2, x11] + tbl v4.16b, {v2.16b}, v4.16b + + b 1f + + .p2align 4 +.Lbegin: + ldr q0, [x0] // load true heads +0: + ldr q4, [x1] +1: + + cmeq v2.16b, v0.16b, #0 // NUL byte present? + cmeq v4.16b, v0.16b, v4.16b // which bytes match? + + orn v2.16b, v2.16b, v4.16b // mismatch or NUL byte? + + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + + cbnz x5, .Lhead_mismatch + + ldr q2, [x8, #16] // load second chunk + ldr q3, [x10, #16] + subs x9, x9, x11 // is a&0xf >= b&0xf + b.lo .Lswapped // if not swap operands + sub x12, x10, x9 + ldr q0, [x12, #16]! + sub x10, x10, x8 + sub x11, x10, x9 + + cmeq v1.16b, v3.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + add x8, x8, #16 + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfound + mvn x5, x5 + cbnz x5, .Lmismatch + add x8, x8, #16 // advance aligned pointers + + /* + * During the main loop, the layout of the two strings is something like: + * + * v ------1------ v ------2------ v + * X0: AAAAAAAAAAAAABBBBBBBBBBBBBBBB... + * X1: AAAAAAAAAAAAABBBBBBBBBBBBBBBBCCC... + * + * where v indicates the alignment boundaries and corresponding chunks + * of the strings have the same letters. Chunk A has been checked in + * the previous iteration. This iteration, we first check that string + * X1 doesn't end within region 2, then we compare chunk B between the + * two strings. As X1 is known not to hold a NUL byte in regions 1 + * and 2 at this point, this also ensures that x0 has not ended yet. + */ + .p2align 4 +0: + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + cmeq v1.16b, v1.16b, #0 // end of string? + cmeq v0.16b, v0.16b, v2.16b // do the chunks match? + + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfound + mvn x5, x5 // any mismatches? + cbnz x5, .Lmismatch + + add x8, x8, #16 + + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + add x8, x8, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfound2 + mvn x5, x5 + cbz x5, 0b + + sub x8, x8, #16 // roll back second increment +.Lmismatch: + rbit x2, x5 + clz x2, x2 // index of mismatch + lsr x2, x2, #2 + add x11, x8, x11 + + ldrb w4, [x8, x2] + ldrb w5, [x11, x2] + sub w0, w4, w5 // byte difference + ret + + .p2align 4 +.Lnulfound2: + sub x8, x8, #16 + +.Lnulfound: + mov x7, x9 + mov x4, x6 + + ubfiz x7, x7, #2, #4 // x7 = (x7 & 0xf) << 2 + lsl x6, x6, x7 // adjust NUL mask to indices + orn x5, x6, x5 + cbnz x5, .Lmismatch + + /* + * (x0) == (x1) and NUL is past the string. + * Compare (x1) with the corresponding part + * of the other string until the NUL byte. + */ + ldr q0, [x8, x9] + ldr q1, [x8, x10] + + cmeq v1.16b, v0.16b, v1.16b + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + orn x5, x4, x5 + + rbit x2, x5 + clz x2, x2 + lsr x5, x2, #2 + + add x10, x10, x8 // restore x10 pointer + add x8, x8, x9 // point to corresponding chunk + + ldrb w4, [x8, x5] + ldrb w5, [x10, x5] + sub w0, w4, w5 + ret + + .p2align 4 +.Lhead_mismatch: + rbit x2, x5 + clz x2, x2 // index of mismatch + lsr x2, x2, #2 + ldrb w4, [x0, x2] + ldrb w5, [x1, x2] + sub w0, w4, w5 + ret + + /* + * If (a&0xf) < (b&0xf), we do the same thing but with swapped + * operands. I found that this performs slightly better than + * using conditional moves to do the swap branchless. + */ + .p2align 4 +.Lswapped: + add x12, x8, x9 + ldr q0, [x12, #16]! + sub x8, x8, x10 + add x11, x8, x9 + neg x9, x9 + + cmeq v1.16b, v2.16b, #0 + cmeq v0.16b, v0.16b, v3.16b + add x10, x10, #16 + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfounds + mvn x5, x5 + cbnz x5, .Lmismatchs + add x10, x10, #16 + + /* + * During the main loop, the layout of the two strings is something like: + * + * v ------1------ v ------2------ v + * X1: AAAAAAAAAAAAABBBBBBBBBBBBBBBB... + * X0: AAAAAAAAAAAAABBBBBBBBBBBBBBBBCCC... + * + * where v indicates the alignment boundaries and corresponding chunks + * of the strings have the same letters. Chunk A has been checked in + * the previous iteration. This iteration, we first check that string + * X0 doesn't end within region 2, then we compare chunk B between the + * two strings. As X0 is known not to hold a NUL byte in regions 1 + * and 2 at this point, this also ensures that X1 has not ended yet. + */ + .p2align 4 +0: + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfounds + mvn x5, x5 + cbnz x5, .Lmismatchs + + add x10, x10, #16 + + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + add x10, x10, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + fmov x6, d1 + shrn v0.8b, v0.8h, #4 + fmov x5, d0 + cbnz x6, .Lnulfound2s + mvn x5, x5 + cbz x5, 0b + + sub x10, x10, #16 + +.Lmismatchs: + rbit x2, x5 + clz x2, x2 + lsr x2, x2, #2 + add x11, x10, x11 + + ldrb w4, [x10, x2] + ldrb w5, [x11, x2] + sub w0, w5, w4 + ret + + .p2align 4 +.Lnulfound2s: + sub x10, x10, #16 +.Lnulfounds: + mov x7, x9 + mov x4, x6 + + ubfiz x7, x7, #2, #4 + lsl x6, x6, x7 + orn x5, x6, x5 + cbnz x5, .Lmismatchs + + ldr q0, [x10, x9] + ldr q1, [x10, x8] + + cmeq v1.16b, v0.16b, v1.16b + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + orn x5, x4, x5 + + rbit x2, x5 + clz x2, x2 + lsr x5, x2, #2 + + add x11, x10, x8 + add x10, x10, x9 + + ldrb w4, [x10, x5] + ldrb w5, [x11, x5] + sub w0, w5, w4 + ret + +END(__strcmp) + + .section .rodata + .p2align 4 +shift_data: + .byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + .fill 16, 1, -1 + .size shift_data, .-shift_data -- 2.45.0 From 4aefba538882a002e4a6d42ea63fad9d34d74243 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:13:54 +0200 Subject: [PATCH 02/15] lib/libc/aarch64/string: add strspn optimized implementation This is a port of the Scalar optimized variant of strspn for amd64 to aarch64. It utilizes a LUT to speed up the function, a SIMD variant is still under development. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46396 --- lib/libc/aarch64/string/Makefile.inc | 4 +- lib/libc/aarch64/string/strspn.S | 111 +++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strspn.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index bb74fec75e4b..5b11c6a015c8 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -21,7 +21,9 @@ AARCH64_STRING_FUNCS= \ MDSRCS+= \ - strcmp.S + strcmp.S \ + strspn.S + # # Add the above functions. Generate an asm file that includes the needed # Arm Optimized Routines file defining the function name to the libc name. diff --git a/lib/libc/aarch64/string/strspn.S b/lib/libc/aarch64/string/strspn.S new file mode 100644 index 000000000000..0ef42c2b737e --- /dev/null +++ b/lib/libc/aarch64/string/strspn.S @@ -0,0 +1,111 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + + .weak strspn + .set strspn, __strspn + .text + +ENTRY(__strspn) + + /* check for special cases */ + ldrb w4, [x1] // first character in set + cbz w4, .Lzero // empty set always returns 0 + + mov x15, #1 // preload register with 1 for stores + + // set is only one character + ldrb w5, [x1, #1] // second character in the set + cbz w5, .Lsingle + + stp x29, x30, [sp, #-16]! + mov x29, sp + sub sp, sp, #256 // allocate 256 bytes on the stack + + /* no special case matches -- prepare lookup table */ + mov w3, #28 +0: add x9, sp, x3, lsl #3 + stp xzr, xzr, [x9] + stp xzr, xzr, [x9, #16] + subs w3, w3, #4 + b.cs 0b + + strb w15, [sp, x4] // register first character in set + add x1, x1, #2 + + /* process remaining chars in set */ + .p2align 4 + + +0: ldrb w4, [x1] // next char in set + strb w15, [sp, x5] // register previous char + cbz w4, 1f // NUL encountered? + + ldrb w5, [x1, #1] + add x1, x1, #2 + strb w15, [sp, x4] + cbnz w5, 0b + +1: mov x5, x0 // stash a copy of src + + /* find mismatch */ + .p2align 4 +0: ldrb w8, [x0] + ldrb w9, [sp, x8] + cbz w9, 2f + + ldrb w8, [x0, #1] + ldrb w9, [sp, x8] + cbz w9, 3f + + ldrb w8, [x0, #2] + ldrb w9, [sp, x8] + cbz w9, 4f + + ldrb w8, [x0, #3] + add x0, x0, #4 + ldrb w9, [sp, x8] + cbnz w9, 0b + + sub x0, x0, #3 +4: sub x5, x5, #1 +3: add x0, x0, #1 +2: sub x0, x0, x5 + mov sp, x29 + ldp x29, x30, [sp], #16 + ret + +.Lzero: + mov x0, #0 + ret + +.Lsingle: + ldrb w8, [x0, x5] + cmp w4, w8 + b.ne 1f + + add x5, x5, #1 + ldrb w8, [x0, x5] + cmp w4, w8 + b.ne 1f + + add x5, x5, #1 + ldrb w8, [x0, x5] + cmp w4, w8 + b.ne 1f + + add x5, x5, #1 + ldrb w8, [x0, x5] + add x5, x5, #1 + cmp w4, w8 + b.eq .Lsingle + + sub x5, x5, #1 +1: mov x0, x5 + ret + +END(__strspn) -- 2.45.0 From 5e831c09c51310fa8ce32798f938371e1cc4baf8 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:01 +0200 Subject: [PATCH 03/15] lib/libc/aarch64/string: add strcspn optimized implementation This is a port of the Scalar optimized variant of strcspn for amd64 to aarch64 It utilizes a LUT to speed up the function, a SIMD variant is still under development. Performance benchmarks are as usual generated by strperf. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46398 --- lib/libc/aarch64/string/Makefile.inc | 3 +- lib/libc/aarch64/string/strcspn.S | 109 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strcspn.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 5b11c6a015c8..597e75b81e35 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -22,7 +22,8 @@ AARCH64_STRING_FUNCS= \ MDSRCS+= \ strcmp.S \ - strspn.S + strspn.S \ + strcspn.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strcspn.S b/lib/libc/aarch64/string/strcspn.S new file mode 100644 index 000000000000..8f2d6d20f0f6 --- /dev/null +++ b/lib/libc/aarch64/string/strcspn.S @@ -0,0 +1,109 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + + .weak strcspn + .set strcspn, __strcspn + .text + +ENTRY(__strcspn) + stp x29, x30, [sp, #-16]! + mov x29, sp + mov x15, #1 // preload register with 1 for stores + + /* check for special cases */ + ldrb w4, [x1] // first character in the set + cbz w4, .Lstrlen + + movi v0.16b, #0 + + ldrb w5, [x1, #1] // second character in the set + cbz w5, .Lstrchr + + sub sp, sp, #256 // allocate 256 bytes on the stack + + /* no special case matches -- prepare lookup table */ + mov w3, #20 + .p2align 4 +0: add x9, sp, x3, lsl #3 + stp xzr, xzr, [x9] + stp xzr, xzr, [x9, #16] + subs w3, w3, #4 + b.cs 0b + + /* utilize SIMD stores to speed up zeroing the table */ + stp q0, q0, [sp, #6*32] + stp q0, q0, [sp, #7*32] + + add x1, x1, #2 + strb w15, [sp, x4] // register first chars in the set + strb w15, [sp, x5] + + mov x4, x0 // stash a copy of src + + /* process remaining chars in set */ + .p2align 4 +0: ldrb w5, [x1] + strb w15, [sp, x5] + cbz w5, 1f // end of set? + + ldrb w5, [x1, #1] + strb w15, [sp, x5] + cbz w5, 1f + + add x1, x1, #2 + b 0b + + /* find match */ + .p2align 4 +1: ldrb w8, [x0] + ldrb w9, [sp, x8] + cbnz w9, 2f + + ldrb w8, [x0, #1] + ldrb w9, [sp, x8] + cbnz w9, 3f + + ldrb w8, [x0, #2] + ldrb w9, [sp, x8] + cbnz w9, 4f + + ldrb w8, [x0, #3] + ldrb w9, [sp, x8] + add x0, x0, #4 + cbz w9, 1b + + sub x0, x0, #3 // fix up return value +4: sub x4, x4, #1 +3: add x0, x0, #1 +2: sub x0, x0, x4 + mov sp, x29 + ldp x29, x30, [sp], #16 // restore sp and lr + ret + + /* set is empty, degrades to strlen */ + .p2align 4 +.Lstrlen: + mov sp, x29 + ldp x29, x30, [sp], #16 // restore sp and lr + b strlen + + /* just one character in set, degrades to strchrnul */ + .p2align 4 +.Lstrchr: + stp x0, x1, [sp, #-16]! + mov x1, x4 + + bl strchrnul + + ldp x18, x17, [sp], #16 // restore stashed src + sub x0, x0, x18 + + ldp x29, x30, [sp], #16 // Restore sp and lr + ret + +END(__strcspn) -- 2.45.0 From 4c038466c902beaa5a7613fa2986b4609ddb1899 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:08 +0200 Subject: [PATCH 04/15] lib/libc/aarch64/string: add optimized strpbrk & strsep implementations These are direct copies from the amd64 string functions using the optimized strcspn from D46398 Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46399 --- lib/libc/aarch64/string/Makefile.inc | 4 +- lib/libc/aarch64/string/strpbrk.c | 43 +++++++++++++++++++++ lib/libc/aarch64/string/strsep.c | 57 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strpbrk.c create mode 100644 lib/libc/aarch64/string/strsep.c diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 597e75b81e35..5777e5dd9408 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -23,7 +23,9 @@ AARCH64_STRING_FUNCS= \ MDSRCS+= \ strcmp.S \ strspn.S \ - strcspn.S + strcspn.S \ + strpbrk.c \ + strsep.c # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strpbrk.c b/lib/libc/aarch64/string/strpbrk.c new file mode 100644 index 000000000000..87f587789991 --- /dev/null +++ b/lib/libc/aarch64/string/strpbrk.c @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Robert Clausecker + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE + */ + +#include + +#include + +size_t __strcspn(const char *, const char *); + +char * +strpbrk(const char *s, const char *charset) +{ + size_t loc; + + loc = __strcspn(s, charset); + + return (s[loc] == '\0' ? NULL : (char *)&s[loc]); +} diff --git a/lib/libc/aarch64/string/strsep.c b/lib/libc/aarch64/string/strsep.c new file mode 100644 index 000000000000..7afd47957aa9 --- /dev/null +++ b/lib/libc/aarch64/string/strsep.c @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Robert Clausecker + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE + */ + +#include +#include + +size_t __strcspn(const char *, const char *); + +/* + * We have a fast strcspn() on aarch64. Use it over a direct + * implementation of strsep for better performance. + */ +char * +strsep(char **stringp, const char *delim) +{ + size_t n; + char *s; + + s = *stringp; + if (s == NULL) + return (NULL); + + n = __strcspn(s, delim); + if (s[n] == '\0') + *stringp = NULL; + else { + s[n] = '\0'; + *stringp = s + n + 1; + } + + return (s); +} -- 2.45.0 From 93bdb6d06dcec85f92c6c19887f843386135f4c1 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:15 +0200 Subject: [PATCH 05/15] lib/libc/aarch64/string: strcat enable use of SIMD Call into SIMD strlen and stpcpy for an optimized strcat. Port of D42600 for amd64. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46417 --- lib/libc/aarch64/string/Makefile.inc | 3 ++- lib/libc/aarch64/string/strcat.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strcat.c diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 5777e5dd9408..47258b8b299b 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -25,7 +25,8 @@ MDSRCS+= \ strspn.S \ strcspn.S \ strpbrk.c \ - strsep.c + strsep.c \ + strcat.c # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strcat.c b/lib/libc/aarch64/string/strcat.c new file mode 100644 index 000000000000..c70875be1c1a --- /dev/null +++ b/lib/libc/aarch64/string/strcat.c @@ -0,0 +1,20 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + +#undef strcat /* _FORTIFY_SOURCE */ + +char * +strcat(char * __restrict s, const char * __restrict append) +{ + char *save = s; + + /* call into SIMD optimized functions */ + stpcpy(s + strlen(s), append); + + return(save); +} -- 2.45.0 From 3bb834a2e29be3ab4febd8ba942390a4ff59f3b6 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:23 +0200 Subject: [PATCH 06/15] lib/libc/aarch64/string: add memcmp SIMD implementation This changeset includes a port of the SIMD implementation of memcmp for amd64 to Aarch64. It also solves an issue with the existing implementation for Aarch64 where the return value is not in accordance with the man page and only returns -1,0 or 1 instead of the byte difference. Performance is better than the existing memcmp implementation borrowed from the Arm Optimized Routines except for long strings. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D45621 --- lib/libc/aarch64/string/Makefile.inc | 5 +- lib/libc/aarch64/string/bcmp.S | 8 + lib/libc/aarch64/string/memcmp.S | 213 +++++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 lib/libc/aarch64/string/bcmp.S create mode 100644 lib/libc/aarch64/string/memcmp.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 47258b8b299b..f3e7384d933e 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -5,7 +5,6 @@ AARCH64_STRING_FUNCS= \ memchr \ - memcmp \ memcpy \ memmove \ memrchr \ @@ -26,7 +25,9 @@ MDSRCS+= \ strcspn.S \ strpbrk.c \ strsep.c \ - strcat.c + strcat.c \ + memcmp.S \ + bcmp.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/bcmp.S b/lib/libc/aarch64/string/bcmp.S new file mode 100644 index 000000000000..46f9cb422f26 --- /dev/null +++ b/lib/libc/aarch64/string/bcmp.S @@ -0,0 +1,8 @@ +/*- + * Written by Mateusz Guzik + * Public domain. + */ + +#define BCMP +#include "memcmp.S" + diff --git a/lib/libc/aarch64/string/memcmp.S b/lib/libc/aarch64/string/memcmp.S new file mode 100644 index 000000000000..7edb654570da --- /dev/null +++ b/lib/libc/aarch64/string/memcmp.S @@ -0,0 +1,213 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include +#include + +#ifdef BCMP +#define memcmp bcmp +#define __memcmp __bcmp +#endif + + .weak memcmp + .set memcmp, __memcmp + .text + +ENTRY(__memcmp) + + mov x8, x0 // store base addresses + mov x9, x1 + cbz x2, .Lnone // 0 length + + /* + * Check if buffer is located at end of page to avoid crossing + * into unmapped page. If so, we load 32 bytes from the end of the + * limit and check the other buffer. + */ + + cmp x2, #32 + b.hi .Lbegin + add x3, x8, #32 + add x4, x9, #32 + eor x3, x3, x8 + eor x4, x4, x9 + + tst w3, #PAGE_SIZE + b.eq 0f + + mov x3, #32 + sub x3, x3, x2 + sub x8, x8, x3 + + /* + * We perform a variable shift in the vector register using TBL, + * a suitable permutation is generated by loading a table of bytes + * with a desired offset. + */ + + adrp x0, shift_table + add x0, x0, :lo12:shift_table + add x0, x0, x3 + ldp q0, q1, [x8] + ldp q4, q5, [x0] // load permutation table + tbl v0.16b, {v0.16b, v1.16b}, v4.16b + tbl v1.16b, {v0.16b, v1.16b}, v5.16b + add x8, x8, x3 // reset pointer to beginning of src + b 1f + +0: + ldp q0, q1, [x8] + +1: + tst w4, #PAGE_SIZE + b.eq 0f + + mov x3, #32 + sub x3, x3, x2 + sub x9, x9, x3 + + ldp q2, q3, [x9] + adrp x0, shift_table + add x0, x0, :lo12:shift_table + add x0, x0, x3 + ldp q4, q5, [x0] + tbl v2.16b, {v2.16b, v3.16b}, v4.16b + tbl v3.16b, {v2.16b, v3.16b}, v5.16b + add x9, x9, x3 + b 1f + + /* + * Compare strings of 1--32 bytes. We do this by loading into two + * vector registers and then doing a quick compare with XOR, UMAXP + * do determine if the first 32 bytes all match. + */ + .p2align 4 +.Lbegin: + ldp q0, q1, [x8] +0: + ldp q2, q3, [x9] +1: + + /* quick check if no matches in first 32 bytes */ + eor v4.16b, v0.16b, v2.16b // v4 = b1(0-15) XOR b2(0-15) + eor v5.16b, v1.16b, v3.16b + umaxp v4.16b, v4.16b, v5.16b + umaxp v4.16b, v4.16b, v4.16b // fill v4 with max value + fmov x6, d4 + cbz x6, .Lloop // if d4 is 0 then all matched + + cmeq v0.16b, v0.16b, v2.16b // do compare between 0-15 b1 vs b2 + cmeq v1.16b, v1.16b, v3.16b // do compare between 16-31 b1 vs b2 + shrn v0.8b, v0.8h, #4 // shift right to fit in x1 + shrn v1.8b, v1.8h, #4 + + fmov x1, d0 + fmov x3, d1 + + mvn x0, x1 // invert for clz + mvn x3, x3 + rbit x1, x0 + rbit x3, x3 + clz x1, x1 + clz x3, x3 + add x3, x3, #64 + cmn x0, #0 // any match in LSB? + csel x0, x3, x1, eq // take x3 if none, else x1 + + lsr x0, x0, #2 + cmp x0, x2 + b.hs .Lnone + ldrb w4, [x8, x0] + ldrb w5, [x9, x0] + sub w0, w4, w5 // get the byte difference + ret + + /* + * Compare strings of 32+ bytes. We introduce special handling if + * theres less than 32 bytes left of the limit. + */ + .p2align 4 +.Lloop: + cmp x2, #64 // next iteration => x2 < 32 ? + b.ls .Llast32 + sub x2, x2, #32 + ldp q0, q1, [x8,#32]! + ldp q2, q3, [x9,#32]! + + eor v4.16b, v0.16b, v2.16b + eor v5.16b, v1.16b, v3.16b + umaxp v4.16b, v4.16b, v5.16b + umaxp v4.16b, v4.16b, v4.16b + fmov x6, d4 + cbnz x6, .Lmatch + + /* main loop unrolled */ + cmp x2, #64 // next iteration => x2 < 32 ? + b.ls .Llast32 + sub x2, x2, #32 + ldp q0, q1, [x8,#32]! + ldp q2, q3, [x9,#32]! + + eor v4.16b, v0.16b, v2.16b + eor v5.16b, v1.16b, v3.16b + umaxp v4.16b, v4.16b, v5.16b + umaxp v4.16b, v4.16b, v4.16b + fmov x6, d4 + cbz x6, .Lloop + b .Lmatch + + /* If 32 bytes left to compare only load 32 bytes from x8,x9 - limit to + * avoid overread */ + .p2align 4 +.Llast32: + sub x2, x2, #32 + cmp x2, #0 + b.le .Lnone + add x8, x8, x2 + add x9, x9, x2 + mov x2, #32 + ldp q0, q1, [x8] + ldp q2, q3, [x9] +.Lmatch: + cmeq v0.16b, v0.16b, v2.16b + cmeq v1.16b, v1.16b, v3.16b + + shrn v0.8b, v0.8h, #4 + shrn v1.8b, v1.8h, #4 + fmov x1, d0 + fmov x3, d1 + + mvn x0, x1 + mvn x3, x3 + rbit x1, x0 + rbit x3, x3 + clz x1, x1 + clz x3, x3 + add x3, x3, #64 + cmn x0, #0 + csel x0, x3, x1, eq + + lsr x0, x0, #2 + cmp x0, x2 + b.hs .Lnone + ldrb w4, [x8, x0] + ldrb w5, [x9, x0] + sub w0, w4, w5 + ret + +.Lnone: + mov x0, #0 + ret + +END(__memcmp) + + .section .rodata + .p2align 4 +shift_table: + .byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + .byte 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + .fill 16, 1, -1 + .size shift_table, .-shift_table -- 2.45.0 From ad84d9ae07ccaaeb7db76c2f22ee506a03eb0306 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:31 +0200 Subject: [PATCH 07/15] lib/libc/aarch64/string: add strlcpy SIMD implementation This changeset includes a port of the SIMD implementation of strlcpy for amd64 to Aarch64. It is based on memccpy (D46170) with some minor differences. Performance is significantly better than the scalar implementation. Benchmark results are as usual generated by the strperf utility written by fuz. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46243 --- lib/libc/aarch64/string/Makefile.inc | 3 +- lib/libc/aarch64/string/strlcpy.S | 316 +++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strlcpy.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index f3e7384d933e..7b870a4fe03f 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -27,7 +27,8 @@ MDSRCS+= \ strsep.c \ strcat.c \ memcmp.S \ - bcmp.S + bcmp.S \ + strlcpy.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strlcpy.S b/lib/libc/aarch64/string/strlcpy.S new file mode 100644 index 000000000000..3859aaca447b --- /dev/null +++ b/lib/libc/aarch64/string/strlcpy.S @@ -0,0 +1,316 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + + .weak strlcpy + .set strlcpy, __strlcpy + .text + +ENTRY(__strlcpy) + subs x2, x2, #1 + b.lo .L0 + + mov x9, x0 // stash copy of dst pointer + bic x10, x1, #0xf // src aligned + and x11, x1, #0xf // src offset + + ldr q1, [x10] + cmeq v1.16b, v1.16b, #0 // NUL found in head? + + mov x8, #-1 // fill register with 0xfff..fff + lsl x12, x11, #2 + lsl x8, x8, x12 // mask of bytes in the string + + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + ands x5, x5, x8 + b.ne .Lhead_nul + + ldr q3, [x10, #16] // load second string chunk + ldr q2, [x1] // load true head + mov x8, #32 + sub x8, x8, x11 + + cmeq v1.16b, v3.16b, #0 // NUL found in second chunk? + + subs x2, x2, x8 + b.ls .Lhead_buf_end + + /* process second chunk */ + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + cbnz x5, .Lsecond_nul + + /* string didn't end in second chunk and neither did buffer */ + ldr q1, [x10, #32] // load next string chunk + str q2, [x0] // deposit head into buffer + sub x0, x0, x11 // adjust x0 + str q3, [x0, #16] // deposit second chunk + add x10, x10, #32 // advance src + add x0, x0, #32 // advance dst + subs x2, x2, #16 // enough left for another round? + b.ls 1f + + /* main loop unrolled twice */ + .p2align 4 +0: + cmeq v2.16b, v1.16b, #0 // NUL found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + + cbnz x5, 3f + + str q1, [x0] + ldr q1, [x10, #16] // load next chunk + + cmp x2, #16 // more than a full chunk left? + b.ls 2f + + add x10, x10, #32 // advance pointers + add x0, x0, #32 + + cmeq v2.16b, v1.16b, #0 // NUL found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + cbnz x5, 4f // process chunk if match + + str q1, [x0, #-16] + ldr q1, [x10] // load next chunk + + subs x2, x2, #32 + b.hi 0b + +1: + sub x10, x10, #16 // undo second advancement + add x2, x2, #16 + sub x0, x0, #16 + + /* 1--16 bytes left in the buffer but string has not ended yet */ +2: + cmeq v2.16b, v1.16b, #0 // NUL found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x4, d2 + + mov x6, #0xf + mov x7, x4 + + lsl x5, x2, #2 // shift 0xf to the limits position + lsl x5, x6, x5 + cmp x2, #16 // dont induce match if limit >=16 + csel x5, x5, xzr, lo + orr x8, x4, x5 // treat limit as if terminator present + + rbit x8, x8 // simulate x86 tzcnt + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + add x0, x0, x8 + + ldr q1, [x10, x8] // load tail + str q1, [x0] // store tail + strb wzr, [x0, #16] + + /* continue to find the end of the string */ + cbnz x7, 1f + + /* we opt for a simpler strlen than the one in libc as the + * cmeq, shrn approach is faster for shorter strings. + */ + .p2align 4 +0: + ldr q1, [x10, #32] + cmeq v1.16b, v1.16b, #0 // bytewise compare against NUL + shrn v1.8b, v1.8h, #4 + fmov x7, d1 + cbnz x7, 2f + + ldr q1, [x10, #48] + cmeq v1.16b, v1.16b, #0 // bytewise compare against NUL + shrn v1.8b, v1.8h, #4 + fmov x7, d1 + add x10, x10, #32 + cbz x7, 0b + +1: sub x10, x10, #16 +2: rbit x8, x7 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + sub x10, x10, x1 + add x0, x10, #32 + add x0, x0, x8 + + ret + +4: + sub x10, x10, #16 // undo second advancement + sub x0, x0, #16 // undo second advancement + + /* string has ended but buffer has not */ +3: + rbit x8, x5 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + add x0, x0, x8 // restore dst pointer + add x10, x10, x8 + + ldr q1, [x10, #-15] + str q1, [x0, #-15] + add x0, x0, #1 + sub x0, x10, x1 + + ret + +.Lhead_buf_end: + shrn v1.8b, v1.8h, #4 + fmov x8, d1 + + add x2, x2, #32 // restore limit + + mov x7, x8 + mov x6, #0xf + + cmp x2, #16 // should we induce a match or not + b.lo 0f + + rbit x8, x8 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + add x8, x8, #16 + + cmp x8, x2 + csel x8, x8, x2, lo // copy min(buflen, srclen) bytes + b 1f +0: + + rbit x8, x8 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + mov x8, x2 +1: + + sub x8, x8, x11 + strb wzr, [x9, x8] + + /* continue to find the end of the string */ + cbnz x7, 1f + + /* we opt for a simpler strlen than the one in libc as the + * cmeq, shrn approach is faster for shorter strings. + */ + .p2align 4 +0: + ldr q1, [x10, #32] + cmeq v1.16b, v1.16b, #0 // bytewise compare against NUL + shrn v1.8b, v1.8h, #4 + fmov x7, d1 + cbnz x7, 2f + + ldr q1, [x10, #48] + cmeq v1.16b, v1.16b, #0 // bytewise compare against NUL + shrn v1.8b, v1.8h, #4 + fmov x7, d1 + add x10, x10, #32 + cbz x7, 0b + +1: sub x10, x10, #16 +2: rbit x6, x7 + clz x6, x6 // index of mismatch + lsr x6, x6, #2 + + sub x10, x10, x1 + add x0, x10, #32 + add x0, x0, x6 + + add x4, x9, x8 // dst + cnt + add x5, x1, x8 // src + cnt + + b .L1732 + +.Lsecond_nul: + add x2, x2, x8 + + rbit x8, x5 + clz x8, x8 // index of mismatch + lsr x5, x8, #2 + + sub x8, x11, #16 + sub x0, x5, x8 // string length + + cmp x0, x2 // did we match or hit limit first? + csel x8, x2, x0, hi + + add x4, x9, x8 // dst + cnt + add x5, x1, x8 // src + cnt + + strb wzr, [x4] + + /* copy 17-32 bytes */ +.L1732: + cmp x8, #16 + b.lo .L0816 + ldp x16, x17, [x1] + ldp x12, x1, [x5, #-16] + stp x16, x17, [x9] + stp x12, x1, [x4, #-16] + ret + +.Lhead_nul: + rbit x8, x5 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + sub x0, x8, x11 + cmp x0, x2 + csel x8, x2, x0, hi + + add x4, x9, x8 // dst + cnt + add x5, x1, x8 // src + cnt + strb wzr, [x4] + + /* Copy 8-16 bytes */ +.L0816: + tbz x8, #3, .L0407 + ldr x16, [x1] + ldr x17, [x5, #-8] + str x16, [x9] + str x17, [x4, #-8] + ret + + /* Copy 4-7 bytes */ + .p2align 4 +.L0407: + cmp x8, #3 + b.ls .L0203 + ldr w16, [x1] + ldr w18, [x5, #-4] + str w16, [x9] + str w18, [x4, #-4] + ret + +.L0203: + tbz x8, 1, .L0001 + ldrh w16, [x1] + ldrh w17, [x5, #-2] + strh w16, [x9] + strh w17, [x4, #-2] + ret + +.L0001: + ldrb w16, [x1] + strb w16, [x9] + strb wzr, [x4] + ret + +.L0: + mov x0, x1 + b strlen + ret +END(__strlcpy) -- 2.45.0 From 6d1d91ed4f592864498b6b1631519553784521a2 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:14:37 +0200 Subject: [PATCH 08/15] lib/libc/aarch64/string: add strncmp SIMD implementation This changeset includes a port of the SIMD implementation of strncmp for amd64 to Aarch64. It is based on D45839 with added handling for the limit. An extended unit test for strncmp is currently being written to make sure the bounds checks for page crossings work as expected. Performance is significantly better than the existing implementation from the Arm Optimized Routines repository. Benchmark results are generated by the strperf utility by fuz. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D45943 --- lib/libc/aarch64/string/Makefile.inc | 4 +- lib/libc/aarch64/string/strncmp.S | 569 +++++++++++++++++++++++++++ 2 files changed, 571 insertions(+), 2 deletions(-) create mode 100644 lib/libc/aarch64/string/strncmp.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 7b870a4fe03f..a1958122bba5 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -14,7 +14,6 @@ AARCH64_STRING_FUNCS= \ strchrnul \ strcpy \ strlen \ - strncmp \ strnlen \ strrchr @@ -28,7 +27,8 @@ MDSRCS+= \ strcat.c \ memcmp.S \ bcmp.S \ - strlcpy.S + strlcpy.S \ + strncmp.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strncmp.S b/lib/libc/aarch64/string/strncmp.S new file mode 100644 index 000000000000..a7f4156da9e8 --- /dev/null +++ b/lib/libc/aarch64/string/strncmp.S @@ -0,0 +1,569 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include +#include + + .weak strncmp + .set strncmp, __strncmp + .text + +ENTRY(__strncmp) + + bic x8, x0, #0xf // x0 aligned to the boundary + and x9, x0, #0xf // x9 is the offset + bic x10, x1, #0xf // x1 aligned to the boundary + and x11, x1, #0xf // x11 is the offset + + subs x2, x2, #1 + b.lo .Lempty + + mov x13, #-1 // save constants for later + mov x16, #0xf + + /* + * Check if either string is located at end of page to avoid crossing + * into unmapped page. If so, we load 16 bytes from the nearest + * alignment boundary and shift based on the offset. + */ + + add x3, x0, #16 // end of head + add x4, x1, #16 + eor x3, x3, x0 + eor x4, x4, x1 // bits that changed + orr x3, x3, x4 // in either str1 or str2 + cmp x2,#16 + b.lo .Llt16 + tbz w3, #PAGE_SHIFT, .Lbegin + + ldr q0, [x8] // load aligned head + ldr q1, [x10] + + lsl x14, x9, #2 + lsl x15, x11, #2 + lsl x3, x13, x14 // string head + lsl x4, x13, x15 + + cmeq v5.16b, v0.16b, #0 + cmeq v6.16b, v1.16b, #0 + + shrn v5.8b, v5.8h, #4 + shrn v6.8b, v6.8h, #4 + fmov x5, d5 + fmov x6, d6 + + adrp x14, shift_data + add x14, x14, :lo12:shift_data + + /* heads may cross page boundary, avoid unmapped loads */ + tst x5, x3 + b.eq 0f + + ldr q4, [x14, x9] // load permutation table + tbl v0.16b, {v0.16b}, v4.16b + + b 1f + .p2align 4 +0: + ldr q0, [x0] // load true head +1: + tst x6, x4 + b.eq 0f + + ldr q4, [x14, x11] + tbl v4.16b, {v1.16b}, v4.16b + + b 1f + + .p2align 4 +.Lbegin: + ldr q0, [x0] // load true heads +0: + ldr q4, [x1] +1: + cmeq v2.16b, v0.16b, #0 // NUL byte present? + cmeq v4.16b, v0.16b, v4.16b // which bytes match? + + orn v2.16b, v2.16b, v4.16b // mismatch or NUL byte? + + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + + cbnz x5, .Lhead_mismatch + /* load head and second chunk */ + ldr q2, [x8, #16] // load second chunk + ldr q3, [x10, #16] + + add x2, x2, x11 + sub x2, x2, #16 + + subs x9, x9, x11 // is a&0xf >= b&0xf + b.lo .Lswapped // if not swap operands + b .Lnormal + + .p2align 4 +.Llt16: + /* + * Check if either string is located at end of page to avoid crossing + * into unmapped page. If so, we load 16 bytes from the nearest + * alignment boundary and shift based on the offset. + */ + tbz w3, #PAGE_SHIFT, 2f + + ldr q0, [x8] // load aligned head + ldr q1, [x10] + + lsl x14, x9, #2 + lsl x15, x11, #2 + lsl x3, x13, x14 // string head + lsl x4, x13, x15 + + /* Introduce a null byte match if the limit is within the aligned chunk */ + add x14, x2, x9 + add x15, x2, x11 + lsl x14, x14, #2 + lsl x15, x15, #2 + lsl x14, x16, x14 + lsl x15, x16, x15 + + cmeq v5.16b, v0.16b, #0 + cmeq v6.16b, v1.16b, #0 + + shrn v5.8b, v5.8h, #4 + shrn v6.8b, v6.8h, #4 + fmov x5, d5 + fmov x6, d6 + + orr x5, x5, x14 // insert match at limit + orr x6, x6, x15 + + adrp x14, shift_data + add x14, x14, :lo12:shift_data + + /* heads may cross page boundary, avoid unmapped loads */ + tst x5, x3 + b.eq 0f + + ldr q4, [x14, x9] // load permutation table + tbl v0.16b, {v0.16b}, v4.16b + + b 1f + .p2align 4 +0: + ldr q0, [x0] // load true head +1: + tst x6, x4 + b.eq 0f + + ldr q4, [x14, x11] + tbl v4.16b, {v1.16b}, v4.16b + + b 1f + + .p2align 4 +2: + ldr q0, [x0] // load true heads +0: + ldr q4, [x1] +1: + + cmeq v2.16b, v0.16b, #0 // NUL byte present? + cmeq v4.16b, v0.16b, v4.16b // which bytes match? + + bic v2.16b, v4.16b, v2.16b // match and not NUL byte + + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + lsl x4, x2, #2 + lsl x4, x13, x4 + orn x5, x4, x5 // mismatch or NUL byte? + +.Lhead_mismatch: + rbit x3, x5 + clz x3, x3 // index of mismatch + lsr x3, x3, #2 + ldrb w4, [x0, x3] + ldrb w5, [x1, x3] + sub w0, w4, w5 + ret + + .p2align 4 +.Lnormal: + sub x12, x10, x9 + ldr q0, [x12, #16]! + sub x10, x10, x8 + sub x11, x10, x9 + + cmeq v1.16b, v3.16b, #0 // NUL present? + cmeq v0.16b, v0.16b, v2.16b // Mismatch between chunks? + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + add x8, x8, #32 // advance to next iteration + + lsl x4, x2, #2 + lsl x4, x13, x4 + orr x3, x6, x4 // introduce a null byte match + cmp x2, #16 // does the buffer end within x2 + csel x6, x3, x6, lo + cbnz x6, .Lnulfound2 // NUL or end of buffer found? + mvn x5, x5 + cbnz x5, .Lmismatch2 + sub x2, x2, #16 + cmp x2, #32 // end of buffer? + b.lo .Ltail + /* + * During the main loop, the layout of the two strings is something like: + * + * v ------1------ v ------2------ v + * X0: AAAAAAAAAAAAABBBBBBBBBBBBBBBB... + * X1: AAAAAAAAAAAAABBBBBBBBBBBBBBBBCCC... + * + * where v indicates the alignment boundaries and corresponding chunks + * of the strings have the same letters. Chunk A has been checked in + * the previous iteration. This iteration, we first check that string + * X1 doesn't end within region 2, then we compare chunk B between the + * two strings. As X1 is known not to hold a NUL byte in regions 1 + * and 2 at this point, this also ensures that x0 has not ended yet. + */ + .p2align 4 +0: + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + cmeq v1.16b, v1.16b, #0 // end of string? + cmeq v0.16b, v0.16b, v2.16b // do the chunks match? + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + cbnz x6, .Lnulfound + mvn x5, x5 // any mismatches? + cbnz x5, .Lmismatch + + add x8, x8, #16 + + /* main loop unrolled twice */ + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + add x8, x8, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + cbnz x6, .Lnulfound2 + mvn x5, x5 + cbnz x5, .Lmismatch2 + sub x2, x2, #32 + cmp x2, #32 // end of buffer? + b.hs 0b // if yes, process tail + + /* end of buffer will occur in next 32 bytes */ +.Ltail: + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + cmeq v1.16b, v1.16b, #0 // end of string? + cmeq v0.16b, v0.16b, v2.16b // do the chunks match? + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + /* + * If x2 <= 16 then we introduce a NUL byte in the + * result from CMEQ to avoid comparing further! + */ + + lsl x4, x2, #2 + lsl x4, x13, x4 + orr x3, x6, x4 // introduce a null byte match + cmp x2, #16 // does the buffer end within x2 + csel x6, x3, x6, lo + + cbnz x6, .Lnulfound // NUL or end of string found + mvn x5, x5 + cbnz x5, .Lmismatch + + add x8, x8, #16 + + /* main loop unrolled twice */ + ldr q0, [x8, x11] + ldr q1, [x8, x10] + ldr q2, [x8] + + add x8, x8, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + ubfiz x4, x2, #2, #4 // (x2 - 16) << 2 + lsl x4, x13, x4 // take first half into account + orr x6, x6, x4 // introduce a null byte match + +.Lnulfound2: + sub x8, x8, #16 + +.Lnulfound: + mov x4, x6 + + ubfiz x7, x9, #2, #4 + lsl x6, x6, x7 // adjust NUL mask to indices + + orn x5, x6, x5 + cbnz x5, .Lmismatch + + /* + * (x0) == (x1) and NUL is past the string. + * Compare (x1) with the corresponding part + * of the other string until the NUL byte. + */ + ldr q0, [x8, x9] + ldr q1, [x8, x10] + + cmeq v1.16b, v0.16b, v1.16b + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + orn x5, x4, x5 + + rbit x3, x5 + clz x3, x3 + lsr x5, x3, #2 + + add x10, x10, x8 // restore x10 pointer + add x8, x8, x9 // point to corresponding chunk + + ldrb w4, [x8, x5] + ldrb w5, [x10, x5] + sub w0, w4, w5 + ret + + .p2align 4 +.Lmismatch2: + sub x8, x8, #16 // roll back second increment +.Lmismatch: + rbit x3, x5 + clz x3, x3 // index of mismatch + lsr x3, x3, #2 + add x11, x8, x11 + + ldrb w4, [x8, x3] + ldrb w5, [x11, x3] + sub w0, w4, w5 // byte difference + ret + + /* + * If (a&0xf) < (b&0xf), we do the same thing but with swapped + * operands. I found that this performs slightly better than + * using conditional moves to do the swap branchless. + */ + .p2align 4 +.Lswapped: + add x12, x8, x9 + ldr q0, [x12, #16]! + sub x8, x8, x10 + add x11, x8, x9 + add x2,x2,x9 + neg x9, x9 + + cmeq v1.16b, v2.16b, #0 + cmeq v0.16b, v0.16b, v3.16b + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + add x10, x10, #32 + + lsl x4, x2, #2 + lsl x4, x13, x4 + orr x3,x6,x4 // introduce a null byte match + cmp x2,#16 + csel x6, x3, x6, lo + cbnz x6, .Lnulfound2s + mvn x5, x5 + cbnz x5, .Lmismatch2s + + sub x2, x2, #16 + cmp x2, #32 + b.lo .Ltails + + /* + * During the main loop, the layout of the two strings is something like: + * + * v ------1------ v ------2------ v + * X1: AAAAAAAAAAAAABBBBBBBBBBBBBBBB... + * X0: AAAAAAAAAAAAABBBBBBBBBBBBBBBBCCC... + * + * where v indicates the alignment boundaries and corresponding chunks + * of the strings have the same letters. Chunk A has been checked in + * the previous iteration. This iteration, we first check that string + * X0 doesn't end within region 2, then we compare chunk B between the + * two strings. As X0 is known not to hold a NUL byte in regions 1 + * and 2 at this point, this also ensures that X1 has not ended yet. + */ + .p2align 4 +0: + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + cbnz x6, .Lnulfounds + mvn x5, x5 + cbnz x5, .Lmismatchs + + add x10, x10, #16 + + /* main loop unrolled twice */ + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + add x10, x10, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + cbnz x6, .Lnulfound2s + mvn x5, x5 + cbnz x5, .Lmismatch2s + sub x2, x2, #32 + cmp x2, #32 + b.hs 0b + +.Ltails: + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + /* + * If x2 <= 16 then we introduce a NUL byte in the + * result from CMEQ to avoid comparing further! + */ + + lsl x4, x2, #2 + lsl x4, x13, x4 + orr x3, x6, x4 // introduce a null byte match + cmp x2, #16 + csel x6, x3, x6, lo + + cbnz x6, .Lnulfounds + mvn x5, x5 + cbnz x5, .Lmismatchs + + add x10, x10, #16 + + ldr q0, [x10, x11] + ldr q1, [x10, x8] + ldr q2, [x10] + + add x10, x10, #16 + cmeq v1.16b, v1.16b, #0 + cmeq v0.16b, v0.16b, v2.16b + + shrn v1.8b, v1.8h, #4 + shrn v0.8b, v0.8h, #4 + fmov x6, d1 + fmov x5, d0 + + ubfiz x4, x2, #2, #4 + lsl x4, x13, x4 + orr x6, x6, x4 // introduce a null byte match + +.Lnulfound2s: + sub x10, x10, #16 +.Lnulfounds: + mov x4, x6 + + ubfiz x7, x9, #2, #4 + lsl x6, x6, x7 + + orn x5, x6, x5 + + cbnz x5, .Lmismatchs + + ldr q0, [x10, x9] + ldr q1, [x10, x8] + + cmeq v1.16b, v0.16b, v1.16b + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + orn x5, x4, x5 + + rbit x3, x5 + clz x3, x3 + lsr x5, x3, #2 + + add x11, x10, x8 + add x10, x10, x9 + + ldrb w4, [x10, x5] + ldrb w5, [x11, x5] + sub w0, w5, w4 + ret + + .p2align 4 +.Lmismatch2s: + sub x10, x10, #16 +.Lmismatchs: + rbit x3, x5 + clz x3, x3 + lsr x3, x3, #2 + add x11, x10, x11 + + ldrb w4, [x10, x3] + ldrb w5, [x11, x3] + sub w0, w5, w4 + ret + + .p2align 4 +.Lempty: + eor x0, x0, x0 + ret + +END(__strncmp) + + .section .rodata + .p2align 4 +shift_data: + .byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + .fill 16, 1, -1 + .size shift_data, .-shift_data -- 2.45.0 From fc36c45d0d4dc2dc3f62dc0166c93c7fff4649e0 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:15:13 +0200 Subject: [PATCH 09/15] lib/libc/aarch64/string: add memccpy SIMD implementation This changeset includes a port of the SIMD implementation of memccpy for amd64 to Aarch64. Performance is significantly better than the scalar implementation except for short strings. Benchmark results are as usual generated by the strperf utility written by fuz. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46170 --- lib/libc/aarch64/string/Makefile.inc | 3 +- lib/libc/aarch64/string/memccpy.S | 271 +++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/memccpy.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index a1958122bba5..1758324ff1cb 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -28,7 +28,8 @@ MDSRCS+= \ memcmp.S \ bcmp.S \ strlcpy.S \ - strncmp.S + strncmp.S \ + memccpy.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/memccpy.S b/lib/libc/aarch64/string/memccpy.S new file mode 100644 index 000000000000..7d9fdb14b84b --- /dev/null +++ b/lib/libc/aarch64/string/memccpy.S @@ -0,0 +1,271 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + + .weak memccpy + .set memccpy, __memccpy + .text + +ENTRY(__memccpy) + subs x3, x3, #1 + b.lo .L0 + + dup v0.16b, w2 + + mov x9, x0 // stash copy of src pointer + bic x10, x1, #0xf // src aligned + and x11, x1, #0xf // src offset + + ldr q1, [x10] + cmeq v1.16b, v1.16b, v0.16b // bytewise compare against src char + + mov x8, #-1 // prepare a 0xfff..fff register + mov x6, #0xf + + lsl x12, x11, #2 + lsl x8, x8, x12 // mask of bytes in the string + + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + sub x12, x11, #32 + adds x12, x12, x3 // distance from alignment boundary - 32 + b.cc .Lrunt // branch if buffer length is 32 or less + + ands x8, x8, x5 + b.eq 0f + + /* match in first chunk */ + rbit x8, x8 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + sub x8, x8, x11 // ... from beginning of the string + + add x0, x0, x8 + add x4, x9, x8 // dst + cnt + add x5, x1, x8 // src + cnt + add x0, x0, #1 + + b .L0816 + +0: + ldr q3, [x10, #16] // load second string chunk + ldr q2, [x1] // load true head + cmeq v1.16b, v3.16b, v0.16b // char found in second chunk? + + /* process second chunk */ + shrn v1.8b, v1.8h, #4 + fmov x5, d1 + + cbz x5, 0f + + /* match in second chunk */ + rbit x8, x5 + clz x8, x8 // index of mismatch + lsr x8, x8, #2 + + sub x11, x11, #16 + sub x8, x8, x11 // adjust for alignment offset + add x0, x0, x8 // return value + add x0, x0, #1 + + add x4, x9, x8 + add x5, x1, x8 + b .L1732 + +0: + /* string didn't end in second chunk and neither did buffer */ + ldr q1, [x10, #32] // load next string chunk + str q2, [x0] // deposit head into buffer + sub x0, x0, x11 // adjust x0 + mov x3, x12 + str q3, [x0, #16] // deposit second chunk + + add x10, x10, #32 // advance src + add x0, x0, #32 // advance dst + subs x3, x3, #16 // enough left for another round? + b.lo 1f + + /* main loop unrolled twice */ + .p2align 4 +0: + cmeq v2.16b, v1.16b, v0.16b // char found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + + cbnz x5, 3f + + str q1, [x0] + ldr q1, [x10, #16] // load next chunk + + cmp x3, #16 // more than a full chunk left? + b.lo 2f + + add x10, x10, #32 // advance pointers + add x0, x0, #32 + + cmeq v2.16b, v1.16b, v0.16b // char found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x5, d2 + cbnz x5, 4f // process chunk if match + + str q1, [x0, #-16] + ldr q1, [x10] // load next chunk + + subs x3, x3, #32 + b.hs 0b + +1: + sub x10, x10, #16 // undo second advancement + add x3, x3, #16 + sub x0, x0, #16 + + /* 1--16 bytes left in the buffer but string has not ended yet */ +2: + cmeq v2.16b, v1.16b, v0.16b // char found in second chunk? + shrn v2.8b, v2.8h, #4 + fmov x4, d2 + + lsl x5, x3, #2 // shift 0xf to the limits position + lsl x5, x6, x5 + orr x8, x4, x5 // insert match in mask at limit + + rbit x8, x8 // simulate x86 tzcnt + clz x7, x8 // index of mismatch + lsr x8, x7, #2 + + lsl x5, x6, x7 // simulate x86 bt with shifted 0xf + + add x8, x8, #1 + add x0, x0, x8 + + ldr q1, [x10, x8] // load tail + str q1, [x0] // store tail + + add x0, x0, #16 + + tst x4, x5 // terminator encountered inside buffer? + csel x0, x0, xzr, ne // if yes, return pointer, else NUL + ret + +4: + sub x10, x10, #16 // undo second advancement + sub x0, x0, #16 // undo second advancement + +3: + rbit x8, x5 + clz x8, x8 // index of mismatch + lsr x3, x8, #2 + + add x0, x0, x3 // restore dst pointer + add x10, x10, x3 + ldr q1, [x10, #-15] + str q1, [x0, #-15] + add x0, x0, #1 + ret + +.Lrunt: + add x13, x11, x3 + + mov x7, x5 // keep a copy of original match mask + + lsl x4, x12, #2 // shift 0xf to the limits position + lsl x4, x6, x4 + + cmp x13, #16 // dont induce match if limit >=16 + csel x4, x4, xzr, lo + orr x5, x5, x4 // insert match in mask at limit + + ands x8, x8, x5 // if match always fall through + b.ne 0f + + ldr q4, [x10, #16] // load second string chunk + cmeq v1.16b, v4.16b, v0.16b // char found in second chunk? + + /* process second chunk */ + shrn v1.8b, v1.8h, #4 + fmov x8, d1 + mov x7, x8 + + lsl x4, x12, #2 + lsl x4, x6, x4 + orr x8, x8, x4 // induce match in upper bytes of mask + + rbit x8, x8 + clz x4, x8 // index of mismatch + lsr x8, x4, #2 + add x8, x8, #16 // no match in first chunk + b 1f + +0: + rbit x8, x8 + clz x4, x8 // index of mismatch + lsr x8, x4, #2 +1: + add x0, x0, x8 // return value if terminator not found + sub x0, x0, x11 + add x0, x0, #1 + + /* check if we encountered a match or the limit first */ + lsl x5, x6, x4 + ands x7, x7, x5 // was the terminator present? + csel x0, xzr, x0, eq // return value based on what we matched + + sub x8, x8, x11 + add x4, x9, x8 // dst + cnt + add x5, x1, x8 // src + cnt + + /* copy 17-32 bytes */ +.L1732: + cmp x8, #16 + b.lo .L0816 + add x5, x5, #1 // ldp offsets are powers of 2 + add x4, x4, #1 + ldp x16, x17, [x1] + ldp x12, x13, [x5, #-16] + stp x16, x17, [x9] + stp x12, x13, [x4, #-16] + ret + + /* Copy 8-16 bytes */ +.L0816: + tbz x8, #3, .L0407 + ldr x16, [x1] + ldr x17, [x5, #-7] + str x16, [x9] + str x17, [x4, #-7] + ret + + /* Copy 4-7 bytes */ + .p2align 4 +.L0407: + cmp x8, #3 + b.lo .L0103 + ldr w16, [x1] + ldr w18, [x5, #-3] + str w16, [x9] + str w18, [x4, #-3] + ret + + /* Copy 1-3 bytes */ + .p2align 4 +.L0103: + lsr x14, x8, #1 + ldrb w16, [x1] + ldrb w15, [x5] + ldrb w18, [x1, x14] + strb w16, [x9] + strb w18, [x9, x14] + strb w15, [x4] + ret + +.L0: + eor x0, x0, x0 + ret + +END(__memccpy) -- 2.45.0 From 5f58ce36a8f8dbd4838452c6444ed10c09f7fcc8 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:15:34 +0200 Subject: [PATCH 10/15] lib/libc/aarch64/string: add strncat SIMD implementation This patch requires D46170 as it depends on strlcpy being labeled __memccpy. It's a direct copy from the amd64 string functions. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46292 --- lib/libc/aarch64/string/Makefile.inc | 3 ++- lib/libc/aarch64/string/strncat.c | 29 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/strncat.c diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 1758324ff1cb..b9f70e1c7ecc 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -29,7 +29,8 @@ MDSRCS+= \ bcmp.S \ strlcpy.S \ strncmp.S \ - memccpy.S + memccpy.S \ + strncat.c # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strncat.c b/lib/libc/aarch64/string/strncat.c new file mode 100644 index 000000000000..33b278ac5e04 --- /dev/null +++ b/lib/libc/aarch64/string/strncat.c @@ -0,0 +1,29 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Robert Clausecker + */ + +#include + +#include + +void *__memccpy(void *restrict, const void *restrict, int, size_t); + +char * +strncat(char *dest, const char *src, size_t n) +{ + size_t len; + char *endptr; + + len = strlen(dest); + endptr = __memccpy(dest + len, src, '\0', n); + + /* avoid an extra branch */ + if (endptr == NULL) + endptr = dest + len + n + 1; + + endptr[-1] = '\0'; + + return (dest); +} -- 2.45.0 From c1e5e85d147b46505ee73f2922e83fe81ceabc73 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 23:10:16 +0200 Subject: [PATCH 11/15] lib/libc/aarch64/string: add strlcat SIMD implementation This patch requires D46243 as it depends on strlcpy being labeled __strlcpy. It's a direct copy from the amd64 string functions using memchr and strlcpy to implement strlcat. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46272 --- lib/libc/aarch64/string/Makefile.inc | 3 ++- lib/libc/aarch64/string/memchr.S | 4 ++++ lib/libc/aarch64/string/strlcat.c | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/memchr.S create mode 100644 lib/libc/aarch64/string/strlcat.c diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index b9f70e1c7ecc..6e15941e757f 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -30,7 +30,8 @@ MDSRCS+= \ strlcpy.S \ strncmp.S \ memccpy.S \ - strncat.c + strncat.c \ + strlcat.c # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/memchr.S b/lib/libc/aarch64/string/memchr.S new file mode 100644 index 000000000000..6d4330d9115e --- /dev/null +++ b/lib/libc/aarch64/string/memchr.S @@ -0,0 +1,4 @@ + .weak memchr + .set memchr, __memchr_aarch64 + +#include "aarch64/memchr.S" diff --git a/lib/libc/aarch64/string/strlcat.c b/lib/libc/aarch64/string/strlcat.c new file mode 100644 index 000000000000..c3c996163ade --- /dev/null +++ b/lib/libc/aarch64/string/strlcat.c @@ -0,0 +1,25 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Robert Clausecker + */ + +#include + +#include + +void *__memchr_aarch64(const void *, int, size_t); +size_t __strlcpy(char *restrict, const char *restrict, size_t); + +size_t +strlcat(char *restrict dst, const char *restrict src, size_t dstsize) +{ + char *loc = __memchr_aarch64(dst, '\0', dstsize); + + if (loc != NULL) { + size_t dstlen = (size_t)(loc - dst); + + return (dstlen + __strlcpy(loc, src, dstsize - dstlen)); + } else + return (dstsize + strlen(src)); +} -- 2.45.0 From a52134b3d01866f2604583039790105143e86f10 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 20:13:44 +0200 Subject: [PATCH 12/15] lib/libc/aarch64/string: add memcpy SIMD implementation I noticed that we have a SIMD optimized memcpy in the arm-optimized-routines in /contrib. This patch ensures we use the SIMD variant as opposed to the Scalar optimized variant. Benchmarks are generated by fuz' strperf utility. See the DR for benchmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46251 --- lib/libc/aarch64/string/memcpy.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/libc/aarch64/string/memcpy.S b/lib/libc/aarch64/string/memcpy.S index f403dd2e42a8..53e860750eb2 100644 --- a/lib/libc/aarch64/string/memcpy.S +++ b/lib/libc/aarch64/string/memcpy.S @@ -1,3 +1,3 @@ -#define __memcpy_aarch64 memcpy -#define __memmove_aarch64 memmove -#include "aarch64/memcpy.S" +#define __memcpy_aarch64_simd memcpy +#define __memmove_aarch64_simd memmove +#include "aarch64/memcpy-advsimd.S" -- 2.45.0 From d1c4d8467d8fd7bc286c895821543012c95c27de Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Mon, 26 Aug 2024 21:54:32 +0200 Subject: [PATCH 13/15] lib/libc/aarch64/string: add strlen SIMD implementation Adds a SIMD enhanced strlen for Aarch64. It takes inspiration from the amd64 implementation but I struggled getting the performance I had hoped for on cores like the Graviton3 when compared to the existing implementation from Arm Optimized Routines. See the DR for bechmark results. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D45623 --- lib/libc/aarch64/string/Makefile.inc | 4 +-- lib/libc/aarch64/string/strlen.S | 46 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 lib/libc/aarch64/string/strlen.S diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index 6e15941e757f..f8938dd7d310 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -13,7 +13,6 @@ AARCH64_STRING_FUNCS= \ strchr \ strchrnul \ strcpy \ - strlen \ strnlen \ strrchr @@ -31,7 +30,8 @@ MDSRCS+= \ strncmp.S \ memccpy.S \ strncat.c \ - strlcat.c + strlcat.c \ + strlen.S # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/strlen.S b/lib/libc/aarch64/string/strlen.S new file mode 100644 index 000000000000..7bfac7f4b1e1 --- /dev/null +++ b/lib/libc/aarch64/string/strlen.S @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Getz Mikalsen +*/ + +#include + + .weak strlen + .set strlen, __strlen + .text + +ENTRY(__strlen) + bic x10, x0, #0xf // aligned src + and x9, x0, #0xf + ldr q0, [x10] + cmeq v0.16b, v0.16b, #0 + shrn v0.8b, v0.8h, #4 + fmov x1, d0 + cbz x9, .Laligned + lsl x2, x0, #2 // get the byte offset + lsr x1, x1, x2 // shift by offset index + cbz x1, .Lloop + rbit x1, x1 + clz x0, x1 + lsr x0, x0, #2 + ret + +.Laligned: + cbnz x1, .Ldone + +.Lloop: + ldr q0, [x10, #16]! + cmeq v0.16b, v0.16b, #0 + shrn v0.8b, v0.8h, #4 // reduce to fit mask in GPR + fcmp d0, #0.0 + b.eq .Lloop + fmov x1, d0 +.Ldone: + sub x0, x10, x0 + rbit x1, x1 // reverse bits as NEON has no ctz + clz x3, x1 + lsr x3, x3, #2 + add x0, x0, x3 + ret +END(__strlen) -- 2.45.0 From 39f67a1252f707e6780d42a10e28567cfa4286c3 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Wed, 28 Aug 2024 15:13:45 +0200 Subject: [PATCH 14/15] lib/libc/aarch64/string: add bcopy & bzero wrapper This patch enabled usage of SIMD enhanced functions to implement bcopy and bzero. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46459 --- lib/libc/aarch64/string/Makefile.inc | 4 +++- lib/libc/aarch64/string/bcopy.c | 14 ++++++++++++++ lib/libc/aarch64/string/bzero.c | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 lib/libc/aarch64/string/bcopy.c create mode 100644 lib/libc/aarch64/string/bzero.c diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc index f8938dd7d310..767a29805c63 100644 --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -31,7 +31,9 @@ MDSRCS+= \ memccpy.S \ strncat.c \ strlcat.c \ - strlen.S + strlen.S \ + bcopy.c \ + bzero.c # # Add the above functions. Generate an asm file that includes the needed diff --git a/lib/libc/aarch64/string/bcopy.c b/lib/libc/aarch64/string/bcopy.c new file mode 100644 index 000000000000..0dee529fb9df --- /dev/null +++ b/lib/libc/aarch64/string/bcopy.c @@ -0,0 +1,14 @@ +/*- + * Public domain. + */ + +#include + +#undef bcopy /* _FORTIFY_SOURCE */ + +void +bcopy(const void *src, void *dst, size_t len) +{ + + memmove(dst, src, len); +} diff --git a/lib/libc/aarch64/string/bzero.c b/lib/libc/aarch64/string/bzero.c new file mode 100644 index 000000000000..d82f3061865b --- /dev/null +++ b/lib/libc/aarch64/string/bzero.c @@ -0,0 +1,14 @@ +/*- + * Public domain. + */ + +#include + +#undef bzero /* _FORTIFY_SOURCE */ + +void +bzero(void *b, size_t len) +{ + + memset(b, 0, len); +} -- 2.45.0 From d090e90c408fc911a7e5518af8174a6845aa4bc1 Mon Sep 17 00:00:00 2001 From: Getz Mikalsen Date: Wed, 28 Aug 2024 15:13:55 +0200 Subject: [PATCH 15/15] share/man/man7/simd.7: document SIMD-enhanced aarch64 functions This documents all the newly ported SIMD-enhanced string functions for the aarch64 platform. Tested by: exprun Approved by: fuz Reviewed by: fuz, emaste Sponsored by: Google LLC (GSoC 2024) Differential Revision: https://reviews.freebsd.org/D46452 --- share/man/man7/simd.7 | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/share/man/man7/simd.7 b/share/man/man7/simd.7 index 877bc77adf4b..1db5a10357bc 100644 --- a/share/man/man7/simd.7 +++ b/share/man/man7/simd.7 @@ -24,7 +24,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE . -.Dd June 7, 2024 +.Dd August 26, 2024 .Dt SIMD 7 .Os .Sh NAME @@ -51,40 +51,40 @@ can be used to override this mechanism. Enhanced functions are present for the following architectures: .Bl -column FUNCTION_________ aarch64_ arm_ amd64_ i386_ ppc64_ -offset indent .It Em FUNCTION Ta Em AARCH64 Ta Em ARM Ta Em AMD64 Ta Em I386 Ta Em PPC64 -.It bcmp Ta Ta Ta S1 Ta S -.It bcopy Ta Ta S Ta S Ta S Ta SV -.It bzero Ta Ta S Ta S Ta S +.It bcmp Ta A Ta Ta S1 Ta S +.It bcopy Ta A Ta S Ta S Ta S Ta SV +.It bzero Ta A Ta S Ta S Ta S .It div Ta Ta Ta S Ta S .It index Ta A Ta Ta S1 .It ldiv Ta Ta Ta S Ta S .It lldiv Ta Ta Ta S .It memchr Ta A Ta Ta S1 .It memcmp Ta A Ta S Ta S1 Ta S -.It memccpy Ta Ta Ta S1 -.It memcpy Ta S Ta S Ta S Ta S Ta SV -.It memmove Ta S Ta S Ta S Ta S Ta SV +.It memccpy Ta A Ta Ta S1 +.It memcpy Ta A Ta S Ta S Ta S Ta SV +.It memmove Ta A Ta S Ta S Ta S Ta SV .It memrchr Ta A Ta Ta S1 .It memset Ta A Ta S Ta S Ta S .It rindex Ta A Ta Ta S1 Ta S .It stpcpy Ta A Ta Ta S1 .It stpncpy Ta Ta Ta S1 -.It strcat Ta Ta Ta S1 Ta S +.It strcat Ta A Ta Ta S1 Ta S .It strchr Ta A Ta Ta S1 Ta S .It strchrnul Ta A Ta Ta S1 -.It strcmp Ta S Ta S Ta S1 Ta S +.It strcmp Ta A Ta S Ta S1 Ta S .It strcpy Ta A Ta Ta S1 Ta S Ta S2 -.It strcspn Ta Ta Ta S2 -.It strlcat Ta Ta Ta S1 -.It strlcpy Ta Ta Ta S1 +.It strcspn Ta S Ta Ta S2 +.It strlcat Ta A Ta Ta S1 +.It strlcpy Ta A Ta Ta S1 .It strlen Ta A Ta S Ta S1 -.It strncat Ta Ta Ta S1 -.It strncmp Ta S Ta S Ta S1 Ta S -.It strncpy Ta Ta Ta S1 Ta Ta S2 +.It strncat Ta A Ta Ta S1 +.It strncmp Ta A Ta S Ta S1 Ta S +.It strncpy Ta A Ta Ta S1 Ta Ta S2 .It strnlen Ta A Ta Ta S1 .It strrchr Ta A Ta Ta S1 Ta S -.It strpbrk Ta Ta Ta S2 -.It strsep Ta Ta Ta S2 -.It strspn Ta Ta Ta S2 +.It strpbrk Ta S Ta Ta S2 +.It strsep Ta S Ta Ta S2 +.It strspn Ta S Ta Ta S2 .It swab Ta Ta Ta Ta S .It timingsafe_bcmp Ta Ta Ta S1 .It timingsafe_memcmp Ta Ta Ta S -- 2.45.0