summaryrefslogtreecommitdiff
path: root/src/pkg/runtime/runtime.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/runtime/runtime.h')
-rw-r--r--src/pkg/runtime/runtime.h360
1 files changed, 293 insertions, 67 deletions
diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index 6f5aea11d..08f43a69b 100644
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -19,9 +19,13 @@ typedef double float64;
#ifdef _64BIT
typedef uint64 uintptr;
typedef int64 intptr;
+typedef int64 intgo; // Go's int
+typedef uint64 uintgo; // Go's uint
#else
typedef uint32 uintptr;
-typedef int32 intptr;
+typedef int32 intptr;
+typedef int32 intgo; // Go's int
+typedef uint32 uintgo; // Go's uint
#endif
/*
@@ -48,44 +52,53 @@ typedef struct G G;
typedef struct Gobuf Gobuf;
typedef union Lock Lock;
typedef struct M M;
+typedef struct P P;
typedef struct Mem Mem;
typedef union Note Note;
typedef struct Slice Slice;
typedef struct Stktop Stktop;
typedef struct String String;
+typedef struct FuncVal FuncVal;
typedef struct SigTab SigTab;
typedef struct MCache MCache;
typedef struct FixAlloc FixAlloc;
typedef struct Iface Iface;
typedef struct Itab Itab;
+typedef struct InterfaceType InterfaceType;
typedef struct Eface Eface;
typedef struct Type Type;
typedef struct ChanType ChanType;
typedef struct MapType MapType;
typedef struct Defer Defer;
+typedef struct DeferChunk DeferChunk;
typedef struct Panic Panic;
typedef struct Hmap Hmap;
typedef struct Hchan Hchan;
typedef struct Complex64 Complex64;
typedef struct Complex128 Complex128;
typedef struct WinCall WinCall;
+typedef struct SEH SEH;
typedef struct Timers Timers;
typedef struct Timer Timer;
+typedef struct GCStats GCStats;
+typedef struct LFNode LFNode;
+typedef struct ParFor ParFor;
+typedef struct ParForThread ParForThread;
+typedef struct CgoMal CgoMal;
/*
- * per-cpu declaration.
- * "extern register" is a special storage class implemented by 6c, 8c, etc.
- * on machines with lots of registers, it allocates a register that will not be
- * used in generated code. on the x86, it allocates a slot indexed by a
- * segment register.
+ * Per-CPU declaration.
*
- * amd64: allocated downwards from R15
- * x86: allocated upwards from 0(GS)
- * arm: allocated downwards from R10
+ * "extern register" is a special storage class implemented by 6c, 8c, etc.
+ * On the ARM, it is an actual register; elsewhere it is a slot in thread-
+ * local storage indexed by a segment register. See zasmhdr in
+ * src/cmd/dist/buildruntime.c for details, and be aware that the linker may
+ * make further OS-specific changes to the compiler's output. For example,
+ * 6l/linux rewrites 0(GS) as -16(FS).
*
- * every C file linked into a Go program must include runtime.h
- * so that the C compiler knows to avoid other uses of these registers.
- * the Go compilers know to avoid them.
+ * Every C file linked into a Go program must include runtime.h so that the
+ * C compiler (6c, 8c, etc.) knows to avoid other uses of these dedicated
+ * registers. The Go compiler (6g, 8g, etc.) knows to avoid them.
*/
extern register G* g;
extern register M* m;
@@ -105,14 +118,34 @@ enum
Grunning,
Gsyscall,
Gwaiting,
- Gmoribund,
+ Gmoribund_unused, // currently unused, but hardcoded in gdb scripts
Gdead,
};
enum
{
+ // P status
+ Pidle,
+ Prunning,
+ Psyscall,
+ Pgcstop,
+ Pdead,
+};
+enum
+{
true = 1,
false = 0,
};
+enum
+{
+ PtrSize = sizeof(void*),
+};
+enum
+{
+ // Per-M stack segment cache size.
+ StackCacheSize = 32,
+ // Global <-> per-M stack segment cache transfer batch size.
+ StackCacheBatch = 16,
+};
/*
* structures
@@ -130,7 +163,12 @@ union Note
struct String
{
byte* str;
- int32 len;
+ intgo len;
+};
+struct FuncVal
+{
+ void (*fn)(void);
+ // variable-size, fn-specific data here
};
struct Iface
{
@@ -156,47 +194,61 @@ struct Complex128
struct Slice
{ // must not move anything
byte* array; // actual data
- uint32 len; // number of elements
- uint32 cap; // allocated number of elements
+ uintgo len; // number of elements
+ uintgo cap; // allocated number of elements
};
struct Gobuf
{
// The offsets of these fields are known to (hard-coded in) libmach.
- byte* sp;
+ uintptr sp;
byte* pc;
G* g;
};
+struct GCStats
+{
+ // the struct must consist of only uint64's,
+ // because it is casted to uint64[].
+ uint64 nhandoff;
+ uint64 nhandoffcnt;
+ uint64 nprocyield;
+ uint64 nosyield;
+ uint64 nsleep;
+};
struct G
{
- byte* stackguard; // cannot move - also known to linker, libmach, runtime/cgo
- byte* stackbase; // cannot move - also known to libmach, runtime/cgo
+ uintptr stackguard; // cannot move - also known to linker, libmach, runtime/cgo
+ uintptr stackbase; // cannot move - also known to libmach, runtime/cgo
Defer* defer;
Panic* panic;
Gobuf sched;
- byte* gcstack; // if status==Gsyscall, gcstack = stackbase to use during gc
- byte* gcsp; // if status==Gsyscall, gcsp = sched.sp to use during gc
- byte* gcguard; // if status==Gsyscall, gcguard = stackguard to use during gc
- byte* stack0;
- byte* entry; // initial function
+ uintptr gcstack; // if status==Gsyscall, gcstack = stackbase to use during gc
+ uintptr gcsp; // if status==Gsyscall, gcsp = sched.sp to use during gc
+ byte* gcpc; // if status==Gsyscall, gcpc = sched.pc to use during gc
+ uintptr gcguard; // if status==Gsyscall, gcguard = stackguard to use during gc
+ uintptr stack0;
+ FuncVal* fnstart; // initial function
G* alllink; // on allg
void* param; // passed parameter on wakeup
int16 status;
- int32 goid;
+ int64 goid;
uint32 selgen; // valid sudog pointer
int8* waitreason; // if status==Gwaiting
G* schedlink;
- bool readyonstop;
bool ispanic;
+ bool issystem;
+ int8 raceignore; // ignore race detection events
M* m; // for debuggers, but offset not hard-coded
M* lockedm;
- M* idlem;
int32 sig;
int32 writenbuf;
byte* writebuf;
+ DeferChunk *dchunk;
+ DeferChunk *dchunknext;
uintptr sigcode0;
uintptr sigcode1;
uintptr sigpc;
uintptr gopc; // pc of go statement that created this goroutine
+ uintptr racectx;
uintptr end[];
};
struct M
@@ -213,43 +265,98 @@ struct M
uintptr cret; // return value from C
uint64 procid; // for debuggers, but offset not hard-coded
G* gsignal; // signal-handling G
- uint32 tls[8]; // thread-local storage (for 386 extern register)
+ uintptr tls[4]; // thread-local storage (for x86 extern register)
+ void (*mstartfn)(void);
G* curg; // current running goroutine
+ P* p; // attached P for executing Go code (nil if not executing Go code)
+ P* nextp;
int32 id;
int32 mallocing;
+ int32 throwing;
int32 gcing;
int32 locks;
int32 nomemprof;
- int32 waitnextg;
int32 dying;
int32 profilehz;
int32 helpgc;
+ bool blockingsyscall;
+ bool spinning;
uint32 fastrand;
- uint64 ncgocall;
- Note havenextg;
- G* nextg;
+ uint64 ncgocall; // number of cgo calls in total
+ int32 ncgo; // number of cgo calls currently in progress
+ CgoMal* cgomal;
+ Note park;
M* alllink; // on allm
M* schedlink;
uint32 machport; // Return address for Mach IPC (OS X)
MCache *mcache;
- FixAlloc *stackalloc;
+ int32 stackinuse;
+ uint32 stackcachepos;
+ uint32 stackcachecnt;
+ void* stackcache[StackCacheSize];
G* lockedg;
- G* idleg;
uintptr createstack[32]; // Stack that created this thread.
uint32 freglo[16]; // D[i] lsb and F[i]
uint32 freghi[16]; // D[i] msb and F[i+16]
uint32 fflag; // floating point compare flags
+ uint32 locked; // tracking for LockOSThread
M* nextwaitm; // next M waiting for lock
uintptr waitsema; // semaphore for parking on locks
uint32 waitsemacount;
uint32 waitsemalock;
+ GCStats gcstats;
+ bool racecall;
+ bool needextram;
+ void* racepc;
+ void (*waitunlockf)(Lock*);
+ Lock* waitlock;
+ uint32 moreframesize_minalloc;
+
+ uintptr settype_buf[1024];
+ uintptr settype_bufsize;
#ifdef GOOS_windows
void* thread; // thread handle
#endif
+#ifdef GOOS_plan9
+ int8* notesig;
+#endif
+ SEH* seh;
uintptr end[];
};
+struct P
+{
+ Lock;
+
+ uint32 status; // one of Pidle/Prunning/...
+ P* link;
+ uint32 tick; // incremented on every scheduler or system call
+ M* m; // back-link to associated M (nil if idle)
+ MCache* mcache;
+
+ // Queue of runnable goroutines.
+ G** runq;
+ int32 runqhead;
+ int32 runqtail;
+ int32 runqsize;
+
+ // Available G's (status == Gdead)
+ G* gfree;
+ int32 gfreecnt;
+
+ byte pad[64];
+};
+
+// The m->locked word holds a single bit saying whether
+// external calls to LockOSThread are in effect, and then a counter
+// of the internal nesting depth of lockOSThread / unlockOSThread.
+enum
+{
+ LockExternal = 1,
+ LockInternal = 2,
+};
+
struct Stktop
{
// The offsets of these fields are known to (hard-coded in) libmach.
@@ -288,8 +395,19 @@ struct Func
uintptr pc0; // starting pc, ln for table
int32 ln0;
int32 frame; // stack frame size
- int32 args; // number of 32-bit in/out args
- int32 locals; // number of 32-bit locals
+ int32 args; // in/out args size
+ int32 locals; // locals size
+};
+
+// layout of Itab known to compilers
+struct Itab
+{
+ InterfaceType* inter;
+ Type* type;
+ Itab* link;
+ int32 bad;
+ int32 unused;
+ void (*fun[])(void);
};
struct WinCall
@@ -301,6 +419,11 @@ struct WinCall
uintptr r2;
uintptr err; // error number
};
+struct SEH
+{
+ void* prev;
+ void* handler;
+};
#ifdef GOOS_windows
enum {
@@ -335,10 +458,46 @@ struct Timer
// a well-behaved function and not block.
int64 when;
int64 period;
- void (*f)(int64, Eface);
+ FuncVal *fv;
Eface arg;
};
+// Lock-free stack node.
+struct LFNode
+{
+ LFNode *next;
+ uintptr pushcnt;
+};
+
+// Parallel for descriptor.
+struct ParFor
+{
+ void (*body)(ParFor*, uint32); // executed for each element
+ uint32 done; // number of idle threads
+ uint32 nthr; // total number of threads
+ uint32 nthrmax; // maximum number of threads
+ uint32 thrseq; // thread id sequencer
+ uint32 cnt; // iteration space [0, cnt)
+ void *ctx; // arbitrary user context
+ bool wait; // if true, wait while all threads finish processing,
+ // otherwise parfor may return while other threads are still working
+ ParForThread *thr; // array of thread descriptors
+ // stats
+ uint64 nsteal;
+ uint64 nstealcnt;
+ uint64 nprocyield;
+ uint64 nosyield;
+ uint64 nsleep;
+};
+
+// Track memory allocated by code not written in Go during a cgo call,
+// so that the garbage collector can see them.
+struct CgoMal
+{
+ CgoMal *next;
+ byte *alloc;
+};
+
/*
* defined macros
* you need super-gopher-guru privilege
@@ -347,6 +506,7 @@ struct Timer
#define nelem(x) (sizeof(x)/sizeof((x)[0]))
#define nil ((void*)0)
#define offsetof(s,m) (uint32)(&(((s*)0)->m))
+#define ROUND(x, n) (((x)+(n)-1)&~((n)-1)) /* all-caps to mark as macro: it evaluates n twice */
/*
* known to compiler
@@ -430,12 +590,19 @@ void runtime·nilintercopy(uintptr, void*, void*);
struct Defer
{
int32 siz;
- bool nofree;
+ bool special; // not part of defer frame
+ bool free; // if special, free when done
byte* argp; // where args were copied from
byte* pc;
- byte* fn;
+ FuncVal* fn;
Defer* link;
- byte args[8]; // padded to actual size
+ void* args[1]; // padded to actual size
+};
+
+struct DeferChunk
+{
+ DeferChunk *prev;
+ uintptr off;
};
/*
@@ -453,16 +620,21 @@ struct Panic
* external data
*/
extern String runtime·emptystring;
-G* runtime·allg;
-G* runtime·lastg;
-M* runtime·allm;
+extern uintptr runtime·zerobase;
+extern G* runtime·allg;
+extern G* runtime·lastg;
+extern M* runtime·allm;
+extern P** runtime·allp;
extern int32 runtime·gomaxprocs;
extern bool runtime·singleproc;
extern uint32 runtime·panicking;
-extern int32 runtime·gcwaiting; // gc is waiting to run
-int8* runtime·goos;
-int32 runtime·ncpu;
+extern uint32 runtime·gcwaiting; // gc is waiting to run
+extern int8* runtime·goos;
+extern int32 runtime·ncpu;
extern bool runtime·iscgo;
+extern void (*runtime·sysargs)(int32, uint8**);
+extern uint32 runtime·maxstring;
+extern uint32 runtime·Hchansize;
/*
* common functions and data
@@ -481,7 +653,8 @@ int32 runtime·charntorune(int32*, uint8*, int32);
#define FLUSH(x) USED(x)
void runtime·gogo(Gobuf*, uintptr);
-void runtime·gogocall(Gobuf*, void(*)(void));
+void runtime·gogocall(Gobuf*, void(*)(void), uintptr);
+void runtime·gogocallfn(Gobuf*, FuncVal*);
void runtime·gosave(Gobuf*);
void runtime·lessstack(void);
void runtime·goargs(void);
@@ -490,7 +663,6 @@ void runtime·goenvs_unix(void);
void* runtime·getu(void);
void runtime·throw(int8*);
void runtime·panicstring(int8*);
-uint32 runtime·rnd(uint32, uint32);
void runtime·prints(int8*);
void runtime·printf(int8*, ...);
byte* runtime·mchr(byte*, byte, byte*);
@@ -499,8 +671,8 @@ void runtime·memmove(void*, void*, uint32);
void* runtime·mal(uintptr);
String runtime·catstring(String, String);
String runtime·gostring(byte*);
-String runtime·gostringn(byte*, int32);
-Slice runtime·gobytes(byte*, int32);
+String runtime·gostringn(byte*, intgo);
+Slice runtime·gobytes(byte*, intgo);
String runtime·gostringnocopy(byte*);
String runtime·gostringw(uint16*);
void runtime·initsig(void);
@@ -512,38 +684,47 @@ void runtime·tracebackothers(G*);
int32 runtime·write(int32, void*, int32);
int32 runtime·mincore(void*, uintptr, byte*);
bool runtime·cas(uint32*, uint32, uint32);
+bool runtime·cas64(uint64*, uint64*, uint64);
bool runtime·casp(void**, void*, void*);
// Don't confuse with XADD x86 instruction,
// this one is actually 'addx', that is, add-and-fetch.
uint32 runtime·xadd(uint32 volatile*, int32);
+uint64 runtime·xadd64(uint64 volatile*, int64);
uint32 runtime·xchg(uint32 volatile*, uint32);
uint32 runtime·atomicload(uint32 volatile*);
void runtime·atomicstore(uint32 volatile*, uint32);
+void runtime·atomicstore64(uint64 volatile*, uint64);
+uint64 runtime·atomicload64(uint64 volatile*);
void* runtime·atomicloadp(void* volatile*);
void runtime·atomicstorep(void* volatile*, void*);
-void runtime·jmpdefer(byte*, void*);
+void runtime·jmpdefer(FuncVal*, void*);
void runtime·exit1(int32);
void runtime·ready(G*);
byte* runtime·getenv(int8*);
int32 runtime·atoi(byte*);
-void runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void));
-void runtime·signalstack(byte*, int32);
+void runtime·newosproc(M *mp, void *stk);
+void runtime·mstart(void);
G* runtime·malg(int32);
void runtime·asminit(void);
+void runtime·mpreinit(M*);
void runtime·minit(void);
+void runtime·unminit(void);
+void runtime·signalstack(byte*, int32);
Func* runtime·findfunc(uintptr);
int32 runtime·funcline(Func*, uintptr);
void* runtime·stackalloc(uint32);
void runtime·stackfree(void*, uintptr);
MCache* runtime·allocmcache(void);
+void runtime·freemcache(MCache*);
void runtime·mallocinit(void);
+void runtime·mprofinit(void);
bool runtime·ifaceeq_c(Iface, Iface);
bool runtime·efaceeq_c(Eface, Eface);
-uintptr runtime·ifacehash(Iface);
-uintptr runtime·efacehash(Eface);
+uintptr runtime·ifacehash(Iface, uintptr);
+uintptr runtime·efacehash(Eface, uintptr);
void* runtime·malloc(uintptr size);
void runtime·free(void *v);
-bool runtime·addfinalizer(void*, void(*fn)(void*), int32);
+bool runtime·addfinalizer(void*, FuncVal *fn, uintptr);
void runtime·runpanic(Panic*);
void* runtime·getcallersp(void*);
int32 runtime·mcount(void);
@@ -551,27 +732,35 @@ int32 runtime·gcount(void);
void runtime·mcall(void(*)(G*));
uint32 runtime·fastrand1(void);
+void runtime·setmg(M*, G*);
+void runtime·newextram(void);
void runtime·exit(int32);
void runtime·breakpoint(void);
void runtime·gosched(void);
-void runtime·tsleep(int64);
+void runtime·park(void(*)(Lock*), Lock*, int8*);
+void runtime·tsleep(int64, int8*);
M* runtime·newm(void);
void runtime·goexit(void);
void runtime·asmcgocall(void (*fn)(void*), void*);
void runtime·entersyscall(void);
+void runtime·entersyscallblock(void);
void runtime·exitsyscall(void);
-G* runtime·newproc1(byte*, byte*, int32, int32, void*);
+G* runtime·newproc1(FuncVal*, byte*, int32, int32, void*);
bool runtime·sigsend(int32 sig);
int32 runtime·callers(int32, uintptr*, int32);
int32 runtime·gentraceback(byte*, byte*, byte*, G*, int32, uintptr*, int32);
int64 runtime·nanotime(void);
void runtime·dopanic(int32);
void runtime·startpanic(void);
+void runtime·unwindstack(G*, byte*);
void runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp);
void runtime·resetcpuprofiler(int32);
void runtime·setcpuprofilerate(void(*)(uintptr*, int32), int32);
void runtime·usleep(uint32);
int64 runtime·cputicks(void);
+int64 runtime·tickspersecond(void);
+void runtime·blockevent(int64, int32);
+extern int64 runtime·blockprofilerate;
#pragma varargck argpos runtime·printf 1
#pragma varargck type "d" int32
@@ -589,7 +778,7 @@ int64 runtime·cputicks(void);
#pragma varargck type "S" String
void runtime·stoptheworld(void);
-void runtime·starttheworld(bool);
+void runtime·starttheworld(void);
extern uint32 runtime·worldsema;
/*
@@ -636,6 +825,27 @@ void runtime·futexsleep(uint32*, uint32, int64);
void runtime·futexwakeup(uint32*, uint32);
/*
+ * Lock-free stack.
+ * Initialize uint64 head to 0, compare with 0 to test for emptiness.
+ * The stack does not keep pointers to nodes,
+ * so they can be garbage collected if there are no other pointers to nodes.
+ */
+void runtime·lfstackpush(uint64 *head, LFNode *node);
+LFNode* runtime·lfstackpop(uint64 *head);
+
+/*
+ * Parallel for over [0, n).
+ * body() is executed for each iteration.
+ * nthr - total number of worker threads.
+ * ctx - arbitrary user context.
+ * if wait=true, threads return from parfor() when all work is done;
+ * otherwise, threads can return while other threads are still finishing processing.
+ */
+ParFor* runtime·parforalloc(uint32 nthrmax);
+void runtime·parforsetup(ParFor *desc, uint32 nthr, uint32 n, void *ctx, bool wait, void (*body)(ParFor*, uint32));
+void runtime·parfordo(ParFor *desc);
+
+/*
* This is consistent across Linux and BSD.
* If a new OS is added that is different, move this to
* $GOOS/$GOARCH/defs.h.
@@ -645,6 +855,9 @@ void runtime·futexwakeup(uint32*, uint32);
/*
* low level C-called
*/
+// for mmap, we only pass the lower 32 bits of file offset to the
+// assembly routine; the higher bits (if required), should be provided
+// by the assembly routine as 0.
uint8* runtime·mmap(byte*, uintptr, int32, int32, int32, uint32);
void runtime·munmap(byte*, uintptr);
void runtime·madvise(byte*, uintptr, int32);
@@ -656,6 +869,7 @@ void* runtime·getcallerpc(void*);
* runtime go-called
*/
void runtime·printbool(bool);
+void runtime·printbyte(int8);
void runtime·printfloat(float64);
void runtime·printint(int64);
void runtime·printiface(Iface);
@@ -667,7 +881,7 @@ void runtime·printuint(uint64);
void runtime·printhex(uint64);
void runtime·printslice(Slice);
void runtime·printcomplex(Complex128);
-void reflect·call(byte*, byte*, uint32);
+void reflect·call(FuncVal*, byte*, uint32);
void runtime·panic(Eface);
void runtime·panicindex(void);
void runtime·panicslice(void);
@@ -708,8 +922,8 @@ void runtime·semrelease(uint32*);
int32 runtime·gomaxprocsfunc(int32 n);
void runtime·procyield(uint32);
void runtime·osyield(void);
-void runtime·LockOSThread(void);
-void runtime·UnlockOSThread(void);
+void runtime·lockOSThread(void);
+void runtime·unlockOSThread(void);
void runtime·mapassign(MapType*, Hmap*, byte*, byte*);
void runtime·mapaccess(MapType*, Hmap*, byte*, byte*, bool*);
@@ -719,13 +933,11 @@ void runtime·mapiterkeyvalue(struct hash_iter*, void*, void*);
Hmap* runtime·makemap_c(MapType*, int64);
Hchan* runtime·makechan_c(ChanType*, int64);
-void runtime·chansend(ChanType*, Hchan*, byte*, bool*);
+void runtime·chansend(ChanType*, Hchan*, byte*, bool*, void*);
void runtime·chanrecv(ChanType*, Hchan*, byte*, bool*, bool*);
-int32 runtime·chanlen(Hchan*);
-int32 runtime·chancap(Hchan*);
-bool runtime·showframe(Func*);
+bool runtime·showframe(Func*, bool);
-void runtime·ifaceE2I(struct InterfaceType*, Eface, Iface*);
+void runtime·ifaceE2I(InterfaceType*, Eface, Iface*);
uintptr runtime·memlimit(void);
@@ -738,3 +950,17 @@ uintptr runtime·memlimit(void);
// is forced to deliver the signal to a thread that's actually running.
// This is a no-op on other systems.
void runtime·setprof(bool);
+
+// float.c
+extern float64 runtime·nan;
+extern float64 runtime·posinf;
+extern float64 runtime·neginf;
+extern uint64 ·nan;
+extern uint64 ·posinf;
+extern uint64 ·neginf;
+#define ISNAN(f) ((f) != (f))
+
+enum
+{
+ UseSpanType = 1,
+};
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-04 03:54:11 +0000