summaryrefslogtreecommitdiff
path: root/src/pkg/runtime/slice.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/runtime/slice.c')
-rw-r--r--src/pkg/runtime/slice.c292
1 files changed, 125 insertions, 167 deletions
diff --git a/src/pkg/runtime/slice.c b/src/pkg/runtime/slice.c
index e5726c93b..354c54c86 100644
--- a/src/pkg/runtime/slice.c
+++ b/src/pkg/runtime/slice.c
@@ -5,22 +5,30 @@
#include "runtime.h"
#include "arch_GOARCH.h"
#include "type.h"
+#include "typekind.h"
#include "malloc.h"
+#include "race.h"
-static int32 debug = 0;
+static bool debug = 0;
-static void makeslice1(SliceType*, int32, int32, Slice*);
-static void growslice1(SliceType*, Slice, int32, Slice *);
- void runtime·copy(Slice to, Slice fm, uintptr width, int32 ret);
+static void makeslice1(SliceType*, intgo, intgo, Slice*);
+static void growslice1(SliceType*, Slice, intgo, Slice *);
+ void runtime·copy(Slice to, Slice fm, uintptr width, intgo ret);
// see also unsafe·NewArray
// makeslice(typ *Type, len, cap int64) (ary []any);
void
runtime·makeslice(SliceType *t, int64 len, int64 cap, Slice ret)
{
- if(len < 0 || (int32)len != len)
+ // NOTE: The len > MaxMem/elemsize check here is not strictly necessary,
+ // but it produces a 'len out of range' error instead of a 'cap out of range' error
+ // when someone does make([]T, bignumber). 'cap out of range' is true too,
+ // but since the cap is only being supplied implicitly, saying len is clearer.
+ // See issue 4085.
+ if(len < 0 || (intgo)len != len || t->elem->size > 0 && len > MaxMem / t->elem->size)
runtime·panicstring("makeslice: len out of range");
- if(cap < len || (int32)cap != cap || t->elem->size > 0 && cap > ((uintptr)-1) / t->elem->size)
+
+ if(cap < len || (intgo)cap != cap || t->elem->size > 0 && cap > MaxMem / t->elem->size)
runtime·panicstring("makeslice: cap out of range");
makeslice1(t, len, cap, &ret);
@@ -35,10 +43,10 @@ runtime·makeslice(SliceType *t, int64 len, int64 cap, Slice ret)
// Dummy word to use as base pointer for make([]T, 0).
// Since you cannot take the address of such a slice,
// you can't tell that they all have the same base pointer.
-static uintptr zerobase;
+uintptr runtime·zerobase;
static void
-makeslice1(SliceType *t, int32 len, int32 cap, Slice *ret)
+makeslice1(SliceType *t, intgo len, intgo cap, Slice *ret)
{
uintptr size;
@@ -47,22 +55,34 @@ makeslice1(SliceType *t, int32 len, int32 cap, Slice *ret)
ret->len = len;
ret->cap = cap;
- if(cap == 0)
- ret->array = (byte*)&zerobase;
+ if(size == 0)
+ ret->array = (byte*)&runtime·zerobase;
else if((t->elem->kind&KindNoPointers))
ret->array = runtime·mallocgc(size, FlagNoPointers, 1, 1);
- else
- ret->array = runtime·mal(size);
+ else {
+ ret->array = runtime·mallocgc(size, 0, 1, 1);
+
+ if(UseSpanType) {
+ if(false) {
+ runtime·printf("new slice [%D]%S: %p\n", (int64)cap, *t->elem->string, ret->array);
+ }
+ runtime·settype(ret->array, (uintptr)t->elem | TypeInfo_Array);
+ }
+ }
}
// appendslice(type *Type, x, y, []T) []T
+#pragma textflag 7
void
runtime·appendslice(SliceType *t, Slice x, Slice y, Slice ret)
{
- int32 m;
+ intgo m;
uintptr w;
+ void *pc;
+ uint8 *p, *q;
m = x.len+y.len;
+ w = t->elem->size;
if(m < x.len)
runtime·throw("append: slice overflow");
@@ -72,30 +92,83 @@ runtime·appendslice(SliceType *t, Slice x, Slice y, Slice ret)
else
ret = x;
- w = t->elem->size;
- runtime·memmove(ret.array + ret.len*w, y.array, y.len*w);
+ if(raceenabled) {
+ // Don't mark read/writes on the newly allocated slice.
+ pc = runtime·getcallerpc(&t);
+ // read x[:len]
+ if(m > x.cap)
+ runtime·racereadrangepc(x.array, x.len*w, w, pc, runtime·appendslice);
+ // read y
+ runtime·racereadrangepc(y.array, y.len*w, w, pc, runtime·appendslice);
+ // write x[len(x):len(x)+len(y)]
+ if(m <= x.cap)
+ runtime·racewriterangepc(ret.array+ret.len*w, y.len*w, w, pc, runtime·appendslice);
+ }
+
+ // A very common case is appending bytes. Small appends can avoid the overhead of memmove.
+ // We can generalize a bit here, and just pick small-sized appends.
+ p = ret.array+ret.len*w;
+ q = y.array;
+ w *= y.len;
+ if(w <= appendCrossover) {
+ if(p <= q || w <= p-q) // No overlap.
+ while(w-- > 0)
+ *p++ = *q++;
+ else {
+ p += w;
+ q += w;
+ while(w-- > 0)
+ *--p = *--q;
+ }
+ } else {
+ runtime·memmove(p, q, w);
+ }
ret.len += y.len;
FLUSH(&ret);
}
// appendstr([]byte, string) []byte
+#pragma textflag 7
void
runtime·appendstr(SliceType *t, Slice x, String y, Slice ret)
{
- int32 m;
+ intgo m;
+ void *pc;
+ uintptr w;
+ uint8 *p, *q;
m = x.len+y.len;
if(m < x.len)
- runtime·throw("append: slice overflow");
+ runtime·throw("append: string overflow");
if(m > x.cap)
growslice1(t, x, m, &ret);
else
ret = x;
- runtime·memmove(ret.array + ret.len, y.str, y.len);
+ if(raceenabled) {
+ // Don't mark read/writes on the newly allocated slice.
+ pc = runtime·getcallerpc(&t);
+ // read x[:len]
+ if(m > x.cap)
+ runtime·racereadrangepc(x.array, x.len, 1, pc, runtime·appendstr);
+ // write x[len(x):len(x)+len(y)]
+ if(m <= x.cap)
+ runtime·racewriterangepc(ret.array+ret.len, y.len, 1, pc, runtime·appendstr);
+ }
+
+ // Small appends can avoid the overhead of memmove.
+ w = y.len;
+ p = ret.array+ret.len;
+ q = y.str;
+ if(w <= appendCrossover) {
+ while(w-- > 0)
+ *p++ = *q++;
+ } else {
+ runtime·memmove(p, q, w);
+ }
ret.len += y.len;
FLUSH(&ret);
}
@@ -106,15 +179,21 @@ void
runtime·growslice(SliceType *t, Slice old, int64 n, Slice ret)
{
int64 cap;
+ void *pc;
if(n < 1)
runtime·panicstring("growslice: invalid n");
cap = old.cap + n;
- if((int32)cap != cap || cap > ((uintptr)-1) / t->elem->size)
+ if((intgo)cap != cap || cap < old.cap || (t->elem->size > 0 && cap > MaxMem/t->elem->size))
runtime·panicstring("growslice: cap out of range");
+ if(raceenabled) {
+ pc = runtime·getcallerpc(&t);
+ runtime·racereadrangepc(old.array, old.len*t->elem->size, t->elem->size, pc, runtime·growslice);
+ }
+
growslice1(t, old, cap, &ret);
FLUSH(&ret);
@@ -128,12 +207,17 @@ runtime·growslice(SliceType *t, Slice old, int64 n, Slice ret)
}
static void
-growslice1(SliceType *t, Slice x, int32 newcap, Slice *ret)
+growslice1(SliceType *t, Slice x, intgo newcap, Slice *ret)
{
- int32 m;
+ intgo m;
m = x.cap;
- if(m == 0)
+
+ // Using newcap directly for m+m < newcap handles
+ // both the case where m == 0 and also the case where
+ // m+m/4 wraps around, in which case the loop
+ // below might never terminate.
+ if(m+m < newcap)
m = newcap;
else {
do {
@@ -147,153 +231,13 @@ growslice1(SliceType *t, Slice x, int32 newcap, Slice *ret)
runtime·memmove(ret->array, x.array, ret->len * t->elem->size);
}
-// sliceslice(old []any, lb uint64, hb uint64, width uint64) (ary []any);
-void
-runtime·sliceslice(Slice old, uint64 lb, uint64 hb, uint64 width, Slice ret)
-{
- if(hb > old.cap || lb > hb) {
- if(debug) {
- runtime·prints("runtime.sliceslice: old=");
- runtime·printslice(old);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; hb=");
- runtime·printint(hb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("\n");
-
- runtime·prints("oldarray: nel=");
- runtime·printint(old.len);
- runtime·prints("; cap=");
- runtime·printint(old.cap);
- runtime·prints("\n");
- }
- runtime·panicslice();
- }
-
- // new array is inside old array
- ret.len = hb - lb;
- ret.cap = old.cap - lb;
- ret.array = old.array + lb*width;
-
- FLUSH(&ret);
-
- if(debug) {
- runtime·prints("runtime.sliceslice: old=");
- runtime·printslice(old);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; hb=");
- runtime·printint(hb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("; ret=");
- runtime·printslice(ret);
- runtime·prints("\n");
- }
-}
-
-// sliceslice1(old []any, lb uint64, width uint64) (ary []any);
-void
-runtime·sliceslice1(Slice old, uint64 lb, uint64 width, Slice ret)
-{
- if(lb > old.len) {
- if(debug) {
- runtime·prints("runtime.sliceslice: old=");
- runtime·printslice(old);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("\n");
-
- runtime·prints("oldarray: nel=");
- runtime·printint(old.len);
- runtime·prints("; cap=");
- runtime·printint(old.cap);
- runtime·prints("\n");
- }
- runtime·panicslice();
- }
-
- // new array is inside old array
- ret.len = old.len - lb;
- ret.cap = old.cap - lb;
- ret.array = old.array + lb*width;
-
- FLUSH(&ret);
-
- if(debug) {
- runtime·prints("runtime.sliceslice: old=");
- runtime·printslice(old);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("; ret=");
- runtime·printslice(ret);
- runtime·prints("\n");
- }
-}
-
-// slicearray(old *any, nel uint64, lb uint64, hb uint64, width uint64) (ary []any);
+// copy(to any, fr any, wid uintptr) int
+#pragma textflag 7
void
-runtime·slicearray(byte* old, uint64 nel, uint64 lb, uint64 hb, uint64 width, Slice ret)
+runtime·copy(Slice to, Slice fm, uintptr width, intgo ret)
{
- if(nel > 0 && old == nil) {
- // crash if old == nil.
- // could give a better message
- // but this is consistent with all the in-line checks
- // that the compiler inserts for other uses.
- *old = 0;
- }
-
- if(hb > nel || lb > hb) {
- if(debug) {
- runtime·prints("runtime.slicearray: old=");
- runtime·printpointer(old);
- runtime·prints("; nel=");
- runtime·printint(nel);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; hb=");
- runtime·printint(hb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("\n");
- }
- runtime·panicslice();
- }
-
- // new array is inside old array
- ret.len = hb-lb;
- ret.cap = nel-lb;
- ret.array = old + lb*width;
-
- FLUSH(&ret);
-
- if(debug) {
- runtime·prints("runtime.slicearray: old=");
- runtime·printpointer(old);
- runtime·prints("; nel=");
- runtime·printint(nel);
- runtime·prints("; lb=");
- runtime·printint(lb);
- runtime·prints("; hb=");
- runtime·printint(hb);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("; ret=");
- runtime·printslice(ret);
- runtime·prints("\n");
- }
-}
+ void *pc;
-// copy(to any, fr any, wid uint32) int
-void
-runtime·copy(Slice to, Slice fm, uintptr width, int32 ret)
-{
if(fm.len == 0 || to.len == 0 || width == 0) {
ret = 0;
goto out;
@@ -303,6 +247,12 @@ runtime·copy(Slice to, Slice fm, uintptr width, int32 ret)
if(to.len < ret)
ret = to.len;
+ if(raceenabled) {
+ pc = runtime·getcallerpc(&to);
+ runtime·racewriterangepc(to.array, ret*width, width, pc, runtime·copy);
+ runtime·racereadrangepc(fm.array, ret*width, width, pc, runtime·copy);
+ }
+
if(ret == 1 && width == 1) { // common case worth about 2x to do here
*to.array = *fm.array; // known to be a byte pointer
} else {
@@ -325,9 +275,12 @@ out:
}
}
+#pragma textflag 7
void
-runtime·slicestringcopy(Slice to, String fm, int32 ret)
+runtime·slicestringcopy(Slice to, String fm, intgo ret)
{
+ void *pc;
+
if(fm.len == 0 || to.len == 0) {
ret = 0;
goto out;
@@ -337,6 +290,11 @@ runtime·slicestringcopy(Slice to, String fm, int32 ret)
if(to.len < ret)
ret = to.len;
+ if(raceenabled) {
+ pc = runtime·getcallerpc(&to);
+ runtime·racewriterangepc(to.array, ret, 1, pc, runtime·slicestringcopy);
+ }
+
runtime·memmove(to.array, fm.str, ret);
out:
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-19 16:44:16 +0000