Pavona Software APIs
memory.c
1// Copyright lowRISC contributors (OpenTitan project).
2// Licensed under the Apache License, Version 2.0, see LICENSE for details.
3// SPDX-License-Identifier: Apache-2.0
4
5#include "sw/device/lib/base/memory.h"
6
7#include <assert.h>
8#include <stdint.h>
9
10#include "sw/device/lib/base/macros.h"
11
12#ifdef OT_PLATFORM_RV32
13#define OT_PREFIX_IF_NOT_RV32(name) name
14#else
15#define OT_PREFIX_IF_NOT_RV32(name) ot_##name
16#endif
17
18static size_t compute_num_leading_bytes(const void *left, const void *right,
19 size_t len) {
20 if (len < alignof(uint32_t)) {
21 return len;
22 }
23 const size_t left_ahead = OT_UNSIGNED(misalignment32_of((uintptr_t)left));
24 const size_t right_ahead = OT_UNSIGNED(misalignment32_of((uintptr_t)right));
25 if (right == NULL || left_ahead == right_ahead) {
26 return (4 - left_ahead) & 0x3;
27 }
28 return len;
29}
30
31/**
32 * Compute the bounds of the word-aligned region for the given buffers.
33 *
34 * It's more efficient for our memory functions to operate on `uint32_t` values
35 * than individual bytes, but we can only read `uint32_t` values from aligned
36 * addresses. This function effectively breaks the given buffers into three
37 * consecutive chunks: the unaligned "head", the aligned "body", and the
38 * unaligned "tail".
39 *
40 * @param[in] left The memory function's first buffer argument. Cannot be NULL.
41 * @param[in] right The memory function's second buffer argument. May be NULL.
42 * @param[in] len The length in bytes of both `left` and `right.`
43 * @param[out] out_body_offset The start of the body region.
44 * @param[out] out_tail_offset The start of the tail region.
45 */
46static void compute_alignment(const void *left, const void *right, size_t len,
47 size_t *out_body_offset,
48 size_t *out_tail_offset) {
49 const size_t num_leading_bytes = compute_num_leading_bytes(left, right, len);
50 *out_body_offset = num_leading_bytes;
51
52 const size_t num_words = (len - num_leading_bytes) / sizeof(uint32_t);
53 *out_tail_offset = num_leading_bytes + num_words * sizeof(uint32_t);
54}
55
56static uint32_t repeat_byte_to_u32(uint8_t byte) {
57 const uint32_t word = byte;
58 return word << 24 | word << 16 | word << 8 | word;
59}
60
61void *OT_PREFIX_IF_NOT_RV32(memcpy)(void *restrict dest,
62 const void *restrict src, size_t len) {
63 unsigned char *dest8 = (unsigned char *)dest;
64 const unsigned char *src8 = (const unsigned char *)src;
65 size_t body_offset, tail_offset;
66 compute_alignment(dest, src, len, &body_offset, &tail_offset);
67 size_t i = 0;
68 for (; i < body_offset; ++i) {
69 dest8[i] = src8[i];
70 }
71 for (; i < tail_offset; i += sizeof(uint32_t)) {
72 uint32_t word = read_32(&src8[i]);
73 write_32(word, &dest8[i]);
74 }
75 for (; i < len; ++i) {
76 dest8[i] = src8[i];
77 }
78 return dest;
79}
80
81void *OT_PREFIX_IF_NOT_RV32(memset)(void *dest, int value, size_t len) {
82 unsigned char *dest8 = (unsigned char *)dest;
83 const uint8_t value8 = (uint8_t)value;
84
85 size_t body_offset, tail_offset;
86 compute_alignment(dest, NULL, len, &body_offset, &tail_offset);
87 size_t i = 0;
88 for (; i < body_offset; ++i) {
89 dest8[i] = value8;
90 }
91 const uint32_t value32 = repeat_byte_to_u32(value8);
92 for (; i < tail_offset; i += sizeof(uint32_t)) {
93 write_32(value32, &dest8[i]);
94 }
95 for (; i < len; ++i) {
96 dest8[i] = value8;
97 }
98 return dest;
99}
100
101enum {
102 kMemCmpEq = 0,
103 kMemCmpLt = -42,
104 kMemCmpGt = 42,
105};
106
107int OT_PREFIX_IF_NOT_RV32(memcmp)(const void *lhs, const void *rhs,
108 size_t len) {
109 const unsigned char *lhs8 = (const unsigned char *)lhs;
110 const unsigned char *rhs8 = (const unsigned char *)rhs;
111 size_t body_offset, tail_offset;
112 compute_alignment(lhs, rhs, len, &body_offset, &tail_offset);
113 size_t i = 0;
114 for (; i < body_offset; ++i) {
115 if (lhs8[i] < rhs8[i]) {
116 return kMemCmpLt;
117 } else if (lhs8[i] > rhs8[i]) {
118 return kMemCmpGt;
119 }
120 }
121 for (; i < tail_offset; i += sizeof(uint32_t)) {
122#if OT_BUILD_FOR_STATIC_ANALYZER
123 assert(&lhs8[i] != NULL);
124 assert(&rhs8[i] != NULL);
125#endif
126 uint32_t word_left = __builtin_bswap32(read_32(&lhs8[i]));
127 uint32_t word_right = __builtin_bswap32(read_32(&rhs8[i]));
128 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__,
129 "memcmp assumes that the system is little endian.");
130 if (word_left < word_right) {
131 return kMemCmpLt;
132 } else if (word_left > word_right) {
133 return kMemCmpGt;
134 }
135 }
136 for (; i < len; ++i) {
137 if (lhs8[i] < rhs8[i]) {
138 return kMemCmpLt;
139 } else if (lhs8[i] > rhs8[i]) {
140 return kMemCmpGt;
141 }
142 }
143 return kMemCmpEq;
144}
145
146int memrcmp(const void *lhs, const void *rhs, size_t len) {
147 const unsigned char *lhs8 = (const unsigned char *)lhs;
148 const unsigned char *rhs8 = (const unsigned char *)rhs;
149 size_t body_offset, tail_offset;
150 compute_alignment(lhs, rhs, len, &body_offset, &tail_offset);
151 size_t end = len;
152 for (; end > tail_offset; --end) {
153 const size_t i = end - 1;
154 if (lhs8[i] < rhs8[i]) {
155 return kMemCmpLt;
156 } else if (lhs8[i] > rhs8[i]) {
157 return kMemCmpGt;
158 }
159 }
160 for (; end > body_offset; end -= sizeof(uint32_t)) {
161 const size_t i = end - sizeof(uint32_t);
162#if OT_BUILD_FOR_STATIC_ANALYZER
163 assert(&lhs8[i] != NULL);
164 assert(&rhs8[i] != NULL);
165#endif
166 uint32_t word_left = read_32(&lhs8[i]);
167 uint32_t word_right = read_32(&rhs8[i]);
168 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__,
169 "memrcmp assumes that the system is little endian.");
170 if (word_left < word_right) {
171 return kMemCmpLt;
172 } else if (word_left > word_right) {
173 return kMemCmpGt;
174 }
175 }
176 for (; end > 0; --end) {
177 const size_t i = end - 1;
178 if (lhs8[i] < rhs8[i]) {
179 return kMemCmpLt;
180 } else if (lhs8[i] > rhs8[i]) {
181 return kMemCmpGt;
182 }
183 }
184 return kMemCmpEq;
185}
186
187void *OT_PREFIX_IF_NOT_RV32(memchr)(const void *ptr, int value, size_t len) {
188 const unsigned char *ptr8 = (const unsigned char *)ptr;
189 const uint8_t value8 = (uint8_t)value;
190
191 size_t body_offset, tail_offset;
192 compute_alignment(ptr, NULL, len, &body_offset, &tail_offset);
193 size_t i = 0;
194 for (; i < body_offset; ++i) {
195 if (ptr8[i] == value8) {
196 return (void *)&ptr8[i];
197 }
198 }
199 const uint32_t value32 = repeat_byte_to_u32(value8);
200 for (; i < tail_offset; i += sizeof(uint32_t)) {
201 uint32_t word = read_32(&ptr8[i]);
202 uint32_t bits_eq = ~(word ^ value32);
203 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__,
204 "memchr assumes that the system is little endian.");
205 if ((bits_eq & UINT8_MAX) == UINT8_MAX) {
206 return (void *)&ptr8[i];
207 }
208 if (((bits_eq >> 8) & UINT8_MAX) == UINT8_MAX) {
209 return (void *)&ptr8[i + 1];
210 }
211 if (((bits_eq >> 16) & UINT8_MAX) == UINT8_MAX) {
212 return (void *)&ptr8[i + 2];
213 }
214 if (((bits_eq >> 24) & UINT8_MAX) == UINT8_MAX) {
215 return (void *)&ptr8[i + 3];
216 }
217 }
218 for (; i < len; ++i) {
219 if (ptr8[i] == value8) {
220 return (void *)&ptr8[i];
221 }
222 }
223 return NULL;
224}
225
226void *OT_PREFIX_IF_NOT_RV32(memrchr)(const void *ptr, int value, size_t len) {
227 const unsigned char *ptr8 = (const unsigned char *)ptr;
228 const uint8_t value8 = (uint8_t)value;
229
230 size_t body_offset, tail_offset;
231 compute_alignment(ptr, NULL, len, &body_offset, &tail_offset);
232
233 size_t end = len;
234 for (; end > tail_offset; --end) {
235 const size_t i = end - 1;
236 if (ptr8[i] == value8) {
237 return (void *)&ptr8[i];
238 }
239 }
240 const uint32_t value32 = repeat_byte_to_u32(value8);
241 for (; end > body_offset; end -= sizeof(uint32_t)) {
242 const size_t i = end - sizeof(uint32_t);
243 uint32_t word = read_32(&ptr8[i]);
244 uint32_t bits_eq = ~(word ^ value32);
245 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__,
246 "memrchr assumes that the system is little endian.");
247 if (((bits_eq >> 24) & UINT8_MAX) == UINT8_MAX) {
248 return (void *)&ptr8[i + 3];
249 }
250 if (((bits_eq >> 16) & UINT8_MAX) == UINT8_MAX) {
251 return (void *)&ptr8[i + 2];
252 }
253 if (((bits_eq >> 8) & UINT8_MAX) == UINT8_MAX) {
254 return (void *)&ptr8[i + 1];
255 }
256 if ((bits_eq & UINT8_MAX) == UINT8_MAX) {
257 return (void *)&ptr8[i];
258 }
259 }
260 for (; end > 0; --end) {
261 const size_t i = end - 1;
262 if (ptr8[i] == value8) {
263 return (void *)&ptr8[i];
264 }
265 }
266 return NULL;
267}
268
269// `extern` declarations to give the inline functions in the corresponding
270// header a link location.
271
272extern ptrdiff_t misalignment32_of(uintptr_t);
273extern uint32_t read_32(const void *);
274extern void write_32(uint32_t, void *);
275extern uint64_t read_64(const void *);
276extern void write_64(uint64_t, void *);
277
278#undef OT_PREFIX_IF_NOT_RV32