Skip to content

Commit 0c6b55e

Browse files
randall77randall77
committedJul 16, 2014
runtime: convert map implementation to Go.
It's a bit slower, but not painfully so. There is still room for improvement (saving space so we can use nosplit, and removing the requirement for hash/eq stubs). benchmark old ns/op new ns/op delta BenchmarkMegMap 23.5 24.2 +2.98% BenchmarkMegOneMap 14.9 15.7 +5.37% BenchmarkMegEqMap 71668 72234 +0.79% BenchmarkMegEmptyMap 4.05 4.93 +21.73% BenchmarkSmallStrMap 21.9 22.5 +2.74% BenchmarkMapStringKeysEight_16 23.1 26.3 +13.85% BenchmarkMapStringKeysEight_32 21.9 25.0 +14.16% BenchmarkMapStringKeysEight_64 21.9 25.1 +14.61% BenchmarkMapStringKeysEight_1M 21.9 25.0 +14.16% BenchmarkIntMap 21.8 12.5 -42.66% BenchmarkRepeatedLookupStrMapKey32 39.3 30.2 -23.16% BenchmarkRepeatedLookupStrMapKey1M 322353 322675 +0.10% BenchmarkNewEmptyMap 129 136 +5.43% BenchmarkMapIter 137 107 -21.90% BenchmarkMapIterEmpty 7.14 8.71 +21.99% BenchmarkSameLengthMap 5.24 6.82 +30.15% BenchmarkBigKeyMap 34.5 35.3 +2.32% BenchmarkBigValMap 36.1 36.1 +0.00% BenchmarkSmallKeyMap 26.9 26.7 -0.74% LGTM=rsc R=golang-codereviews, dave, dvyukov, rsc, gobot, khr CC=golang-codereviews https://golang.org/cl/99380043
·
go1.24.3go1.4beta1
1 parent 3b1b840 commit 0c6b55e

29 files changed

+1971
-1520
lines changed
 

‎src/cmd/api/goapi.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,14 @@ func (w *Walker) parseFile(dir, file string) (*ast.File, error) {
370370
log.Fatalf("incorrect generated file: %s", err)
371371
}
372372
}
373+
if w.context != nil && file == fmt.Sprintf("zruntime_defs_%s_%s.go", w.context.GOOS, w.context.GOARCH) {
374+
// Just enough to keep the api checker happy.
375+
src := "package runtime; type maptype struct{}; type _type struct{}; type alg struct{}"
376+
f, err = parser.ParseFile(fset, filename, src, 0)
377+
if err != nil {
378+
log.Fatalf("incorrect generated file: %s", err)
379+
}
380+
}
373381

374382
if f == nil {
375383
f, err = parser.ParseFile(fset, filename, nil, 0)
@@ -488,6 +496,11 @@ func (w *Walker) Import(name string) (pkg *types.Package) {
488496
if !contains(filenames, n) {
489497
filenames = append(filenames, n)
490498
}
499+
500+
n = fmt.Sprintf("zruntime_defs_%s_%s.go", w.context.GOOS, w.context.GOARCH)
501+
if !contains(filenames, n) {
502+
filenames = append(filenames, n)
503+
}
491504
}
492505

493506
// Parse package files.

‎src/cmd/dist/buildruntime.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,8 @@ mkzasm(char *dir, char *file)
231231
aggr = "cbctxt";
232232
else if(streq(fields.p[1], "SEH"))
233233
aggr = "seh";
234+
else if(streq(fields.p[1], "Alg"))
235+
aggr = "alg";
234236
}
235237
if(hasprefix(lines.p[i], "}"))
236238
aggr = nil;

‎src/cmd/ld/dwarf.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1222,7 +1222,7 @@ synthesizemaptypes(DWDie *die)
12221222
DWAttr *a;
12231223

12241224
hash = walktypedef(defgotype(lookup_or_diag("type.runtime.hmap")));
1225-
bucket = walktypedef(defgotype(lookup_or_diag("type.runtime.bucket")));
1225+
bucket = walktypedef(defgotype(lookup_or_diag("type.runtime.bmap")));
12261226

12271227
if (hash == nil)
12281228
return;

‎src/pkg/reflect/asm_386.s

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
2525
MOVL CX, 4(SP)
2626
CALL ·callMethod(SB)
2727
RET
28+
29+
// Stubs to give reflect package access to runtime services
30+
// TODO: should probably be done another way.
31+
TEXT ·makemap(SB),NOSPLIT,$0-0
32+
JMP runtime·reflect_makemap(SB)
33+
TEXT ·mapaccess(SB),NOSPLIT,$0-0
34+
JMP runtime·reflect_mapaccess(SB)
35+
TEXT ·mapassign(SB),NOSPLIT,$0-0
36+
JMP runtime·reflect_mapassign(SB)
37+
TEXT ·mapdelete(SB),NOSPLIT,$0-0
38+
JMP runtime·reflect_mapdelete(SB)
39+
TEXT ·mapiterinit(SB),NOSPLIT,$0-0
40+
JMP runtime·reflect_mapiterinit(SB)
41+
TEXT ·mapiterkey(SB),NOSPLIT,$0-0
42+
JMP runtime·reflect_mapiterkey(SB)
43+
TEXT ·mapiternext(SB),NOSPLIT,$0-0
44+
JMP runtime·reflect_mapiternext(SB)
45+
TEXT ·maplen(SB),NOSPLIT,$0-0
46+
JMP runtime·reflect_maplen(SB)
47+
TEXT ·ismapkey(SB),NOSPLIT,$0-0
48+
JMP runtime·reflect_ismapkey(SB)

‎src/pkg/reflect/asm_amd64.s

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16
2525
MOVQ CX, 8(SP)
2626
CALL ·callMethod(SB)
2727
RET
28+
29+
// Stubs to give reflect package access to runtime services
30+
// TODO: should probably be done another way.
31+
TEXT ·makemap(SB),NOSPLIT,$0-0
32+
JMP runtime·reflect_makemap(SB)
33+
TEXT ·mapaccess(SB),NOSPLIT,$0-0
34+
JMP runtime·reflect_mapaccess(SB)
35+
TEXT ·mapassign(SB),NOSPLIT,$0-0
36+
JMP runtime·reflect_mapassign(SB)
37+
TEXT ·mapdelete(SB),NOSPLIT,$0-0
38+
JMP runtime·reflect_mapdelete(SB)
39+
TEXT ·mapiterinit(SB),NOSPLIT,$0-0
40+
JMP runtime·reflect_mapiterinit(SB)
41+
TEXT ·mapiterkey(SB),NOSPLIT,$0-0
42+
JMP runtime·reflect_mapiterkey(SB)
43+
TEXT ·mapiternext(SB),NOSPLIT,$0-0
44+
JMP runtime·reflect_mapiternext(SB)
45+
TEXT ·maplen(SB),NOSPLIT,$0-0
46+
JMP runtime·reflect_maplen(SB)
47+
TEXT ·ismapkey(SB),NOSPLIT,$0-0
48+
JMP runtime·reflect_ismapkey(SB)

‎src/pkg/reflect/asm_amd64p32.s

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
2525
MOVL CX, 4(SP)
2626
CALL ·callMethod(SB)
2727
RET
28+
29+
// Stubs to give reflect package access to runtime services
30+
// TODO: should probably be done another way.
31+
TEXT ·makemap(SB),NOSPLIT,$0-0
32+
JMP runtime·reflect_makemap(SB)
33+
TEXT ·mapaccess(SB),NOSPLIT,$0-0
34+
JMP runtime·reflect_mapaccess(SB)
35+
TEXT ·mapassign(SB),NOSPLIT,$0-0
36+
JMP runtime·reflect_mapassign(SB)
37+
TEXT ·mapdelete(SB),NOSPLIT,$0-0
38+
JMP runtime·reflect_mapdelete(SB)
39+
TEXT ·mapiterinit(SB),NOSPLIT,$0-0
40+
JMP runtime·reflect_mapiterinit(SB)
41+
TEXT ·mapiterkey(SB),NOSPLIT,$0-0
42+
JMP runtime·reflect_mapiterkey(SB)
43+
TEXT ·mapiternext(SB),NOSPLIT,$0-0
44+
JMP runtime·reflect_mapiternext(SB)
45+
TEXT ·maplen(SB),NOSPLIT,$0-0
46+
JMP runtime·reflect_maplen(SB)
47+
TEXT ·ismapkey(SB),NOSPLIT,$0-0
48+
JMP runtime·reflect_ismapkey(SB)

‎src/pkg/reflect/asm_arm.s

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
2525
MOVW R1, 8(R13)
2626
BL ·callMethod(SB)
2727
RET
28+
29+
// Stubs to give reflect package access to runtime services
30+
// TODO: should probably be done another way.
31+
TEXT ·makemap(SB),NOSPLIT,$-4-0
32+
B runtime·reflect_makemap(SB)
33+
TEXT ·mapaccess(SB),NOSPLIT,$-4-0
34+
B runtime·reflect_mapaccess(SB)
35+
TEXT ·mapassign(SB),NOSPLIT,$-4-0
36+
B runtime·reflect_mapassign(SB)
37+
TEXT ·mapdelete(SB),NOSPLIT,$-4-0
38+
B runtime·reflect_mapdelete(SB)
39+
TEXT ·mapiterinit(SB),NOSPLIT,$-4-0
40+
B runtime·reflect_mapiterinit(SB)
41+
TEXT ·mapiterkey(SB),NOSPLIT,$-4-0
42+
B runtime·reflect_mapiterkey(SB)
43+
TEXT ·mapiternext(SB),NOSPLIT,$-4-0
44+
B runtime·reflect_mapiternext(SB)
45+
TEXT ·maplen(SB),NOSPLIT,$-4-0
46+
B runtime·reflect_maplen(SB)
47+
TEXT ·ismapkey(SB),NOSPLIT,$-4-0
48+
B runtime·reflect_ismapkey(SB)

‎src/pkg/runtime/alg.goc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,8 @@ runtime·nohash(uintptr *h, uintptr s, void *a)
425425
runtime·panicstring("hash of unhashable type");
426426
}
427427

428+
extern uintptr runtime·nohashcode;
429+
428430
void
429431
runtime·noequal(bool *eq, uintptr s, void *a, void *b)
430432
{
@@ -471,6 +473,7 @@ byte runtime·aeskeysched[HashRandomBytes];
471473
void
472474
runtime·hashinit(void)
473475
{
476+
runtime·nohashcode = (uintptr)runtime·nohash;
474477
if(NaCl)
475478
return;
476479

‎src/pkg/runtime/asm_386.s

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1107,6 +1107,14 @@ TEXT runtime·memeq(SB),NOSPLIT,$0-12
11071107
MOVL count+8(FP), BX
11081108
JMP runtime·memeqbody(SB)
11091109

1110+
TEXT runtime·gomemeq(SB),NOSPLIT,$0-13
1111+
MOVL a+0(FP), SI
1112+
MOVL b+4(FP), DI
1113+
MOVL size+8(FP), BX
1114+
CALL runtime·memeqbody(SB)
1115+
MOVB AX, ret+12(FP)
1116+
RET
1117+
11101118
// eqstring tests whether two strings are equal.
11111119
// See runtime_test.go:eqstring_generic for
11121120
// equivlaent Go code.
@@ -2197,3 +2205,81 @@ TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
21972205

21982206
TEXT runtime·timenow(SB), NOSPLIT, $0-0
21992207
JMP time·now(SB)
2208+
2209+
TEXT runtime·fastrand2(SB), NOSPLIT, $0-4
2210+
get_tls(CX)
2211+
MOVL g(CX), AX
2212+
MOVL g_m(AX), AX
2213+
MOVL m_fastrand(AX), DX
2214+
ADDL DX, DX
2215+
MOVL DX, BX
2216+
XORL $0x88888eef, DX
2217+
CMOVLMI BX, DX
2218+
MOVL DX, m_fastrand(AX)
2219+
MOVL DX, ret+0(FP)
2220+
RET
2221+
2222+
// The gohash and goeq trampolines are necessary while we have
2223+
// both Go and C calls to alg functions. Once we move all call
2224+
// sites to Go, we can redo the hash/eq functions to use the
2225+
// Go calling convention and remove these.
2226+
2227+
// convert call to:
2228+
// func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
2229+
// to:
2230+
// func (hash *uintptr, size uintptr, p unsafe.Pointer)
2231+
TEXT runtime·gohash(SB), NOSPLIT, $12-20
2232+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
2233+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
2234+
MOVL a+0(FP), AX
2235+
MOVL alg_hash(AX), AX
2236+
MOVL p+4(FP), CX
2237+
MOVL size+8(FP), DX
2238+
MOVL seed+12(FP), DI
2239+
MOVL DI, ret+16(FP)
2240+
LEAL ret+16(FP), SI
2241+
MOVL SI, 0(SP)
2242+
MOVL DX, 4(SP)
2243+
MOVL CX, 8(SP)
2244+
PCDATA $PCDATA_StackMapIndex, $0
2245+
CALL *AX
2246+
RET
2247+
2248+
DATA gcargs_gohash<>+0x00(SB)/4, $1 // 1 stackmap
2249+
DATA gcargs_gohash<>+0x04(SB)/4, $10 // 5 args
2250+
DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
2251+
GLOBL gcargs_gohash<>(SB),RODATA,$12
2252+
2253+
DATA gclocals_gohash<>+0x00(SB)/4, $1 // 1 stackmap
2254+
DATA gclocals_gohash<>+0x04(SB)/4, $0 // 0 locals
2255+
GLOBL gclocals_gohash<>(SB),RODATA,$8
2256+
2257+
// convert call to:
2258+
// func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
2259+
// to:
2260+
// func (eq *bool, size uintptr, p, q unsafe.Pointer)
2261+
TEXT runtime·goeq(SB), NOSPLIT, $16-17
2262+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
2263+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
2264+
MOVL alg+0(FP), AX
2265+
MOVL alg_equal(AX), AX
2266+
MOVL p+4(FP), CX
2267+
MOVL q+8(FP), DX
2268+
MOVL size+12(FP), DI
2269+
LEAL ret+16(FP), SI
2270+
MOVL SI, 0(SP)
2271+
MOVL DI, 4(SP)
2272+
MOVL CX, 8(SP)
2273+
MOVL DX, 12(SP)
2274+
PCDATA $PCDATA_StackMapIndex, $0
2275+
CALL *AX
2276+
RET
2277+
2278+
DATA gcargs_goeq<>+0x00(SB)/4, $1 // 1 stackmap
2279+
DATA gcargs_goeq<>+0x04(SB)/4, $10 // 5 args
2280+
DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
2281+
GLOBL gcargs_goeq<>(SB),RODATA,$12
2282+
2283+
DATA gclocals_goeq<>+0x00(SB)/4, $1 // 1 stackmap
2284+
DATA gclocals_goeq<>+0x04(SB)/4, $0 // 0 locals
2285+
GLOBL gclocals_goeq<>(SB),RODATA,$8

‎src/pkg/runtime/asm_amd64.s

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,6 +1075,14 @@ TEXT runtime·memeq(SB),NOSPLIT,$0-24
10751075
MOVQ count+16(FP), BX
10761076
JMP runtime·memeqbody(SB)
10771077

1078+
TEXT runtime·gomemeq(SB),NOSPLIT,$0-25
1079+
MOVQ a+0(FP), SI
1080+
MOVQ b+8(FP), DI
1081+
MOVQ size+16(FP), BX
1082+
CALL runtime·memeqbody(SB)
1083+
MOVB AX, ret+24(FP)
1084+
RET
1085+
10781086
// eqstring tests whether two strings are equal.
10791087
// See runtime_test.go:eqstring_generic for
10801088
// equivlaent Go code.
@@ -2236,3 +2244,81 @@ TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
22362244

22372245
TEXT runtime·timenow(SB), NOSPLIT, $0-0
22382246
JMP time·now(SB)
2247+
2248+
TEXT runtime·fastrand2(SB), NOSPLIT, $0-4
2249+
get_tls(CX)
2250+
MOVQ g(CX), AX
2251+
MOVQ g_m(AX), AX
2252+
MOVL m_fastrand(AX), DX
2253+
ADDL DX, DX
2254+
MOVL DX, BX
2255+
XORL $0x88888eef, DX
2256+
CMOVLMI BX, DX
2257+
MOVL DX, m_fastrand(AX)
2258+
MOVL DX, ret+0(FP)
2259+
RET
2260+
2261+
// The gohash and goeq trampolines are necessary while we have
2262+
// both Go and C calls to alg functions. Once we move all call
2263+
// sites to Go, we can redo the hash/eq functions to use the
2264+
// Go calling convention and remove these.
2265+
2266+
// convert call to:
2267+
// func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
2268+
// to:
2269+
// func (hash *uintptr, size uintptr, p unsafe.Pointer)
2270+
TEXT runtime·gohash(SB), NOSPLIT, $24-40
2271+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
2272+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
2273+
MOVQ a+0(FP), AX
2274+
MOVQ alg_hash(AX), AX
2275+
MOVQ p+8(FP), CX
2276+
MOVQ size+16(FP), DX
2277+
MOVQ seed+24(FP), DI
2278+
MOVQ DI, ret+32(FP)
2279+
LEAQ ret+32(FP), SI // TODO: go vet complains here: "invalid LEAQ of ret+32(FP); bool is 1-byte value"
2280+
MOVQ SI, 0(SP)
2281+
MOVQ DX, 8(SP)
2282+
MOVQ CX, 16(SP)
2283+
PCDATA $PCDATA_StackMapIndex, $0
2284+
CALL *AX
2285+
RET
2286+
2287+
DATA gcargs_gohash<>+0x00(SB)/4, $1 // 1 stackmap
2288+
DATA gcargs_gohash<>+0x04(SB)/4, $10 // 5 args
2289+
DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
2290+
GLOBL gcargs_gohash<>(SB),RODATA,$12
2291+
2292+
DATA gclocals_gohash<>+0x00(SB)/4, $1 // 1 stackmap
2293+
DATA gclocals_gohash<>+0x04(SB)/4, $0 // 0 locals
2294+
GLOBL gclocals_gohash<>(SB),RODATA,$8
2295+
2296+
// convert call to:
2297+
// func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
2298+
// to:
2299+
// func (eq *bool, size uintptr, p, q unsafe.Pointer)
2300+
TEXT runtime·goeq(SB), NOSPLIT, $32-33
2301+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
2302+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
2303+
MOVQ alg+0(FP), AX
2304+
MOVQ alg_equal(AX), AX
2305+
MOVQ p+8(FP), CX
2306+
MOVQ q+16(FP), DX
2307+
MOVQ size+24(FP), DI
2308+
LEAQ ret+32(FP), SI
2309+
MOVQ SI, 0(SP)
2310+
MOVQ DI, 8(SP)
2311+
MOVQ CX, 16(SP)
2312+
MOVQ DX, 24(SP)
2313+
PCDATA $PCDATA_StackMapIndex, $0
2314+
CALL *AX
2315+
RET
2316+
2317+
DATA gcargs_goeq<>+0x00(SB)/4, $1 // 1 stackmap
2318+
DATA gcargs_goeq<>+0x04(SB)/4, $10 // 5 args
2319+
DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
2320+
GLOBL gcargs_goeq<>(SB),RODATA,$12
2321+
2322+
DATA gclocals_goeq<>+0x00(SB)/4, $1 // 1 stackmap
2323+
DATA gclocals_goeq<>+0x04(SB)/4, $0 // 0 locals
2324+
GLOBL gclocals_goeq<>(SB),RODATA,$8

‎src/pkg/runtime/asm_amd64p32.s

Lines changed: 88 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -653,8 +653,8 @@ TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
653653
RET
654654

655655
TEXT runtime·memclr(SB),NOSPLIT,$0-8
656-
MOVL addr+0(FP), DI
657-
MOVL count+4(FP), CX
656+
MOVL ptr+0(FP), DI
657+
MOVL n+4(FP), CX
658658
MOVQ CX, BX
659659
ANDQ $7, BX
660660
SHRQ $3, CX
@@ -730,6 +730,14 @@ TEXT runtime·memeq(SB),NOSPLIT,$0-12
730730
MOVL count+8(FP), BX
731731
JMP runtime·memeqbody(SB)
732732

733+
TEXT runtime·gomemeq(SB),NOSPLIT,$0-13
734+
MOVL a+0(FP), SI
735+
MOVL b+4(FP), DI
736+
MOVL size+8(FP), BX
737+
CALL runtime·memeqbody(SB)
738+
MOVB AX, ret+12(FP)
739+
RET
740+
733741
// eqstring tests whether two strings are equal.
734742
// See runtime_test.go:eqstring_generic for
735743
// equivlaent Go code.
@@ -1108,3 +1116,81 @@ eqret:
11081116

11091117
TEXT runtime·timenow(SB), NOSPLIT, $0-0
11101118
JMP time·now(SB)
1119+
1120+
TEXT runtime·fastrand2(SB), NOSPLIT, $0-4
1121+
get_tls(CX)
1122+
MOVL g(CX), AX
1123+
MOVL g_m(AX), AX
1124+
MOVL m_fastrand(AX), DX
1125+
ADDL DX, DX
1126+
MOVL DX, BX
1127+
XORL $0x88888eef, DX
1128+
CMOVLMI BX, DX
1129+
MOVL DX, m_fastrand(AX)
1130+
MOVL DX, ret+0(FP)
1131+
RET
1132+
1133+
// The gohash and goeq trampolines are necessary while we have
1134+
// both Go and C calls to alg functions. Once we move all call
1135+
// sites to Go, we can redo the hash/eq functions to use the
1136+
// Go calling convention and remove these.
1137+
1138+
// convert call to:
1139+
// func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
1140+
// to:
1141+
// func (hash *uintptr, size uintptr, p unsafe.Pointer)
1142+
TEXT runtime·gohash(SB), NOSPLIT, $12-20
1143+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
1144+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
1145+
MOVL a+0(FP), AX
1146+
MOVL alg_hash(AX), AX
1147+
MOVL p+4(FP), CX
1148+
MOVL size+8(FP), DX
1149+
MOVL seed+12(FP), DI
1150+
MOVL DI, ret+16(FP)
1151+
LEAL ret+16(FP), SI
1152+
MOVL SI, 0(SP)
1153+
MOVL DX, 4(SP)
1154+
MOVL CX, 8(SP)
1155+
PCDATA $PCDATA_StackMapIndex, $0
1156+
CALL *AX
1157+
RET
1158+
1159+
DATA gcargs_gohash<>+0x00(SB)/4, $1 // 1 stackmap
1160+
DATA gcargs_gohash<>+0x04(SB)/4, $10 // 5 args
1161+
DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
1162+
GLOBL gcargs_gohash<>(SB),RODATA,$12
1163+
1164+
DATA gclocals_gohash<>+0x00(SB)/4, $1 // 1 stackmap
1165+
DATA gclocals_gohash<>+0x04(SB)/4, $0 // 0 locals
1166+
GLOBL gclocals_gohash<>(SB),RODATA,$8
1167+
1168+
// convert call to:
1169+
// func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
1170+
// to:
1171+
// func (eq *bool, size uintptr, p, q unsafe.Pointer)
1172+
TEXT runtime·goeq(SB), NOSPLIT, $16-17
1173+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
1174+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
1175+
MOVL alg+0(FP), AX
1176+
MOVL alg_equal(AX), AX
1177+
MOVL p+4(FP), CX
1178+
MOVL q+8(FP), DX
1179+
MOVL size+12(FP), DI
1180+
LEAL ret+16(FP), SI
1181+
MOVL SI, 0(SP)
1182+
MOVL DI, 4(SP)
1183+
MOVL CX, 8(SP)
1184+
MOVL DX, 12(SP)
1185+
PCDATA $PCDATA_StackMapIndex, $0
1186+
CALL *AX
1187+
RET
1188+
1189+
DATA gcargs_goeq<>+0x00(SB)/4, $1 // 1 stackmap
1190+
DATA gcargs_goeq<>+0x04(SB)/4, $10 // 5 args
1191+
DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
1192+
GLOBL gcargs_goeq<>(SB),RODATA,$12
1193+
1194+
DATA gclocals_goeq<>+0x00(SB)/4, $1 // 1 stackmap
1195+
DATA gclocals_goeq<>+0x04(SB)/4, $0 // 0 locals
1196+
GLOBL gclocals_goeq<>(SB),RODATA,$8

‎src/pkg/runtime/asm_arm.s

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,25 @@ _next:
670670
MOVW $0, R0
671671
RET
672672

673+
TEXT runtime·gomemeq(SB),NOSPLIT,$-4-13
674+
MOVW a+0(FP), R1
675+
MOVW b+4(FP), R2
676+
MOVW size+8(FP), R3
677+
ADD R1, R3, R6
678+
MOVW $1, R0
679+
MOVB R0, ret+12(FP)
680+
_next2:
681+
CMP R1, R6
682+
RET.EQ
683+
MOVBU.P 1(R1), R4
684+
MOVBU.P 1(R2), R5
685+
CMP R4, R5
686+
BEQ _next2
687+
688+
MOVW $0, R0
689+
MOVB R0, ret+12(FP)
690+
RET
691+
673692
// eqstring tests whether two strings are equal.
674693
// See runtime_test.go:eqstring_generic for
675694
// equivlaent Go code.
@@ -1190,3 +1209,77 @@ TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
11901209
MOVW.P 4(R1), R0
11911210
MOVW.P R0, 4(R2)
11921211
RET
1212+
1213+
TEXT runtime·fastrand2(SB), NOSPLIT, $-4-4
1214+
MOVW g_m(g), R1
1215+
MOVW m_fastrand(R1), R0
1216+
ADD.S R0, R0
1217+
EOR.MI $0x88888eef, R0
1218+
MOVW R0, m_fastrand(R1)
1219+
MOVW R0, ret+0(FP)
1220+
RET
1221+
1222+
// The gohash and goeq trampolines are necessary while we have
1223+
// both Go and C calls to alg functions. Once we move all call
1224+
// sites to Go, we can redo the hash/eq functions to use the
1225+
// Go calling convention and remove these.
1226+
1227+
// convert call to:
1228+
// func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
1229+
// to:
1230+
// func (hash *uintptr, size uintptr, p unsafe.Pointer)
1231+
TEXT runtime·gohash(SB), NOSPLIT, $12-20
1232+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
1233+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
1234+
MOVW a+0(FP), R0
1235+
MOVW alg_hash(R0), R0
1236+
MOVW p+4(FP), R1
1237+
MOVW size+8(FP), R2
1238+
MOVW seed+12(FP), R3
1239+
MOVW R3, ret+16(FP)
1240+
ADD $36, R13, R4
1241+
MOVW R4, 4(R13)
1242+
MOVW R2, 8(R13)
1243+
MOVW R1, 12(R13)
1244+
PCDATA $PCDATA_StackMapIndex, $0
1245+
BL (R0)
1246+
RET
1247+
1248+
DATA gcargs_gohash<>+0x00(SB)/4, $1 // 1 stackmap
1249+
DATA gcargs_gohash<>+0x04(SB)/4, $10 // 5 args
1250+
DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
1251+
GLOBL gcargs_gohash<>(SB),RODATA,$12
1252+
1253+
DATA gclocals_gohash<>+0x00(SB)/4, $1 // 1 stackmap
1254+
DATA gclocals_gohash<>+0x04(SB)/4, $0 // 0 locals
1255+
GLOBL gclocals_gohash<>(SB),RODATA,$8
1256+
1257+
// convert call to:
1258+
// func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
1259+
// to:
1260+
// func (eq *bool, size uintptr, p, q unsafe.Pointer)
1261+
TEXT runtime·goeq(SB), NOSPLIT, $16-17
1262+
FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
1263+
FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
1264+
MOVW alg+0(FP), R0
1265+
MOVW alg_equal(R0), R0
1266+
MOVW p+4(FP), R1
1267+
MOVW q+8(FP), R2
1268+
MOVW size+12(FP), R3
1269+
ADD $40, R13, R4
1270+
MOVW R4, 4(R13)
1271+
MOVW R3, 8(R13)
1272+
MOVW R2, 12(R13)
1273+
MOVW R1, 16(R13)
1274+
PCDATA $PCDATA_StackMapIndex, $0
1275+
BL (R0)
1276+
RET
1277+
1278+
DATA gcargs_goeq<>+0x00(SB)/4, $1 // 1 stackmap
1279+
DATA gcargs_goeq<>+0x04(SB)/4, $10 // 5 args
1280+
DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
1281+
GLOBL gcargs_goeq<>(SB),RODATA,$12
1282+
1283+
DATA gclocals_goeq<>+0x00(SB)/4, $1 // 1 stackmap
1284+
DATA gclocals_goeq<>+0x04(SB)/4, $0 // 0 locals
1285+
GLOBL gclocals_goeq<>(SB),RODATA,$8

‎src/pkg/runtime/defs.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@
1010
#include "malloc.h"
1111
#include "type.h"
1212
#include "race.h"
13-
#include "hashmap.h"
1413
#include "chan.h"
14+
#include "mprof.h"

‎src/pkg/runtime/export_test.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ var BytesHash = bytesHash
8080
var Int32Hash = int32Hash
8181
var Int64Hash = int64Hash
8282

83-
var hashLoad float64 // declared in hashmap.c
8483
var HashLoad = &hashLoad
8584

8685
func memclrBytes(b []byte)

‎src/pkg/runtime/hashmap.go

Lines changed: 965 additions & 0 deletions
Large diffs are not rendered by default.

‎src/pkg/runtime/hashmap.goc

Lines changed: 0 additions & 1078 deletions
This file was deleted.

‎src/pkg/runtime/hashmap.h

Lines changed: 0 additions & 147 deletions
This file was deleted.

‎src/pkg/runtime/hashmap_fast.c

Lines changed: 0 additions & 233 deletions
This file was deleted.

‎src/pkg/runtime/hashmap_fast.go

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
// Copyright 2014 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package runtime
6+
7+
import (
8+
"unsafe"
9+
)
10+
11+
func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
12+
if raceenabled && h != nil {
13+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
14+
fn := mapaccess1_fast32
15+
pc := **(**uintptr)(unsafe.Pointer(&fn))
16+
racereadpc(unsafe.Pointer(h), callerpc, pc)
17+
}
18+
if h == nil || h.count == 0 {
19+
return unsafe.Pointer(t.elem.zero)
20+
}
21+
var b *bmap
22+
if h.B == 0 {
23+
// One-bucket table. No need to hash.
24+
b = (*bmap)(h.buckets)
25+
} else {
26+
hash := gohash(t.key.alg, unsafe.Pointer(&key), 4, uintptr(h.hash0))
27+
m := uintptr(1)<<h.B - 1
28+
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
29+
if c := h.oldbuckets; c != nil {
30+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
31+
if !evacuated(oldb) {
32+
b = oldb
33+
}
34+
}
35+
}
36+
for {
37+
for i := uintptr(0); i < bucketCnt; i++ {
38+
k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
39+
if k != key {
40+
continue
41+
}
42+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
43+
if t == empty {
44+
continue
45+
}
46+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(h.valuesize))
47+
}
48+
b = b.overflow
49+
if b == nil {
50+
return unsafe.Pointer(t.elem.zero)
51+
}
52+
}
53+
}
54+
55+
func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
56+
if raceenabled && h != nil {
57+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
58+
fn := mapaccess2_fast32
59+
pc := **(**uintptr)(unsafe.Pointer(&fn))
60+
racereadpc(unsafe.Pointer(h), callerpc, pc)
61+
}
62+
if h == nil || h.count == 0 {
63+
return unsafe.Pointer(t.elem.zero), false
64+
}
65+
var b *bmap
66+
if h.B == 0 {
67+
// One-bucket table. No need to hash.
68+
b = (*bmap)(h.buckets)
69+
} else {
70+
hash := gohash(t.key.alg, unsafe.Pointer(&key), 4, uintptr(h.hash0))
71+
m := uintptr(1)<<h.B - 1
72+
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
73+
if c := h.oldbuckets; c != nil {
74+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
75+
if !evacuated(oldb) {
76+
b = oldb
77+
}
78+
}
79+
}
80+
for {
81+
for i := uintptr(0); i < bucketCnt; i++ {
82+
k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
83+
if k != key {
84+
continue
85+
}
86+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
87+
if t == empty {
88+
continue
89+
}
90+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(h.valuesize)), true
91+
}
92+
b = b.overflow
93+
if b == nil {
94+
return unsafe.Pointer(t.elem.zero), false
95+
}
96+
}
97+
}
98+
99+
func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
100+
if raceenabled && h != nil {
101+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
102+
fn := mapaccess1_fast64
103+
pc := **(**uintptr)(unsafe.Pointer(&fn))
104+
racereadpc(unsafe.Pointer(h), callerpc, pc)
105+
}
106+
if h == nil || h.count == 0 {
107+
return unsafe.Pointer(t.elem.zero)
108+
}
109+
var b *bmap
110+
if h.B == 0 {
111+
// One-bucket table. No need to hash.
112+
b = (*bmap)(h.buckets)
113+
} else {
114+
hash := gohash(t.key.alg, unsafe.Pointer(&key), 8, uintptr(h.hash0))
115+
m := uintptr(1)<<h.B - 1
116+
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
117+
if c := h.oldbuckets; c != nil {
118+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
119+
if !evacuated(oldb) {
120+
b = oldb
121+
}
122+
}
123+
}
124+
for {
125+
for i := uintptr(0); i < bucketCnt; i++ {
126+
k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
127+
if k != key {
128+
continue
129+
}
130+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
131+
if t == empty {
132+
continue
133+
}
134+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(h.valuesize))
135+
}
136+
b = b.overflow
137+
if b == nil {
138+
return unsafe.Pointer(t.elem.zero)
139+
}
140+
}
141+
}
142+
143+
func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
144+
if raceenabled && h != nil {
145+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
146+
fn := mapaccess2_fast64
147+
pc := **(**uintptr)(unsafe.Pointer(&fn))
148+
racereadpc(unsafe.Pointer(h), callerpc, pc)
149+
}
150+
if h == nil || h.count == 0 {
151+
return unsafe.Pointer(t.elem.zero), false
152+
}
153+
var b *bmap
154+
if h.B == 0 {
155+
// One-bucket table. No need to hash.
156+
b = (*bmap)(h.buckets)
157+
} else {
158+
hash := gohash(t.key.alg, unsafe.Pointer(&key), 8, uintptr(h.hash0))
159+
m := uintptr(1)<<h.B - 1
160+
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
161+
if c := h.oldbuckets; c != nil {
162+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
163+
if !evacuated(oldb) {
164+
b = oldb
165+
}
166+
}
167+
}
168+
for {
169+
for i := uintptr(0); i < bucketCnt; i++ {
170+
k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
171+
if k != key {
172+
continue
173+
}
174+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
175+
if t == empty {
176+
continue
177+
}
178+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(h.valuesize)), true
179+
}
180+
b = b.overflow
181+
if b == nil {
182+
return unsafe.Pointer(t.elem.zero), false
183+
}
184+
}
185+
}
186+
187+
func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
188+
if raceenabled && h != nil {
189+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
190+
fn := mapaccess1_faststr
191+
pc := **(**uintptr)(unsafe.Pointer(&fn))
192+
racereadpc(unsafe.Pointer(h), callerpc, pc)
193+
}
194+
if h == nil || h.count == 0 {
195+
return unsafe.Pointer(t.elem.zero)
196+
}
197+
key := (*stringStruct)(unsafe.Pointer(&ky))
198+
if h.B == 0 {
199+
// One-bucket table.
200+
b := (*bmap)(h.buckets)
201+
if key.len < 32 {
202+
// short key, doing lots of comparisons is ok
203+
for i := uintptr(0); i < bucketCnt; i++ {
204+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
205+
if t == empty {
206+
continue
207+
}
208+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
209+
if k.len != key.len {
210+
continue
211+
}
212+
if k.str == key.str || gomemeq(k.str, key.str, uintptr(key.len)) {
213+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize))
214+
}
215+
}
216+
return unsafe.Pointer(t.elem.zero)
217+
}
218+
// long key, try not to do more comparisons than necessary
219+
keymaybe := uintptr(bucketCnt)
220+
for i := uintptr(0); i < bucketCnt; i++ {
221+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
222+
if t == empty {
223+
continue
224+
}
225+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
226+
if k.len != key.len {
227+
continue
228+
}
229+
if k.str == key.str {
230+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize))
231+
}
232+
// check first 4 bytes
233+
// TODO: on amd64/386 at least, make this compile to one 4-byte comparison instead of
234+
// four 1-byte comparisons.
235+
if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) {
236+
continue
237+
}
238+
// check last 4 bytes
239+
if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) {
240+
continue
241+
}
242+
if keymaybe != bucketCnt {
243+
// Two keys are potential matches. Use hash to distinguish them.
244+
goto dohash
245+
}
246+
keymaybe = i
247+
}
248+
if keymaybe != bucketCnt {
249+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*ptrSize))
250+
if gomemeq(k.str, key.str, uintptr(key.len)) {
251+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+keymaybe*uintptr(h.valuesize))
252+
}
253+
}
254+
return unsafe.Pointer(t.elem.zero)
255+
}
256+
dohash:
257+
hash := gohash(t.key.alg, unsafe.Pointer(&ky), 2*ptrSize, uintptr(h.hash0))
258+
m := uintptr(1)<<h.B - 1
259+
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
260+
if c := h.oldbuckets; c != nil {
261+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
262+
if !evacuated(oldb) {
263+
b = oldb
264+
}
265+
}
266+
top := uint8(hash >> (ptrSize*8 - 8))
267+
if top < minTopHash {
268+
top += minTopHash
269+
}
270+
for {
271+
for i := uintptr(0); i < bucketCnt; i++ {
272+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
273+
if t != top {
274+
continue
275+
}
276+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
277+
if k.len != key.len {
278+
continue
279+
}
280+
if k.str == key.str || gomemeq(k.str, key.str, uintptr(key.len)) {
281+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize))
282+
}
283+
}
284+
b = b.overflow
285+
if b == nil {
286+
return unsafe.Pointer(t.elem.zero)
287+
}
288+
}
289+
}
290+
291+
func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
292+
if raceenabled && h != nil {
293+
callerpc := gogetcallerpc(unsafe.Pointer(&t))
294+
fn := mapaccess2_faststr
295+
pc := **(**uintptr)(unsafe.Pointer(&fn))
296+
racereadpc(unsafe.Pointer(h), callerpc, pc)
297+
}
298+
if h == nil || h.count == 0 {
299+
return unsafe.Pointer(t.elem.zero), false
300+
}
301+
key := (*stringStruct)(unsafe.Pointer(&ky))
302+
if h.B == 0 {
303+
// One-bucket table.
304+
b := (*bmap)(h.buckets)
305+
if key.len < 32 {
306+
// short key, doing lots of comparisons is ok
307+
for i := uintptr(0); i < bucketCnt; i++ {
308+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
309+
if t == empty {
310+
continue
311+
}
312+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
313+
if k.len != key.len {
314+
continue
315+
}
316+
if k.str == key.str || gomemeq(k.str, key.str, uintptr(key.len)) {
317+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize)), true
318+
}
319+
}
320+
return unsafe.Pointer(t.elem.zero), false
321+
}
322+
// long key, try not to do more comparisons than necessary
323+
keymaybe := uintptr(bucketCnt)
324+
for i := uintptr(0); i < bucketCnt; i++ {
325+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
326+
if t == empty {
327+
continue
328+
}
329+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
330+
if k.len != key.len {
331+
continue
332+
}
333+
if k.str == key.str {
334+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize)), true
335+
}
336+
// check first 4 bytes
337+
if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) {
338+
continue
339+
}
340+
// check last 4 bytes
341+
if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) {
342+
continue
343+
}
344+
if keymaybe != bucketCnt {
345+
// Two keys are potential matches. Use hash to distinguish them.
346+
goto dohash
347+
}
348+
keymaybe = i
349+
}
350+
if keymaybe != bucketCnt {
351+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*ptrSize))
352+
if gomemeq(k.str, key.str, uintptr(key.len)) {
353+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+keymaybe*uintptr(h.valuesize)), true
354+
}
355+
}
356+
return unsafe.Pointer(t.elem.zero), false
357+
}
358+
dohash:
359+
hash := gohash(t.key.alg, unsafe.Pointer(&ky), 2*ptrSize, uintptr(h.hash0))
360+
m := uintptr(1)<<h.B - 1
361+
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(h.bucketsize)))
362+
if c := h.oldbuckets; c != nil {
363+
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(h.bucketsize)))
364+
if !evacuated(oldb) {
365+
b = oldb
366+
}
367+
}
368+
top := uint8(hash >> (ptrSize*8 - 8))
369+
if top < minTopHash {
370+
top += minTopHash
371+
}
372+
for {
373+
for i := uintptr(0); i < bucketCnt; i++ {
374+
t := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
375+
if t != top {
376+
continue
377+
}
378+
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*ptrSize))
379+
if k.len != key.len {
380+
continue
381+
}
382+
if k.str == key.str || gomemeq(k.str, key.str, uintptr(key.len)) {
383+
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(h.valuesize)), true
384+
}
385+
}
386+
b = b.overflow
387+
if b == nil {
388+
return unsafe.Pointer(t.elem.zero), false
389+
}
390+
}
391+
}

‎src/pkg/runtime/malloc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ struct SpecialFinalizer
398398
};
399399

400400
// The described object is being heap profiled.
401-
typedef struct Bucket Bucket; // from mprof.goc
401+
typedef struct Bucket Bucket; // from mprof.h
402402
typedef struct SpecialProfile SpecialProfile;
403403
struct SpecialProfile
404404
{

‎src/pkg/runtime/memmove_386.s

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929

3030
TEXT runtime·memmove(SB), NOSPLIT, $0-12
3131
MOVL to+0(FP), DI
32-
MOVL fr+4(FP), SI
32+
MOVL from+4(FP), SI
3333
MOVL n+8(FP), BX
3434

3535
// REP instructions have a high startup cost, so we handle small sizes

‎src/pkg/runtime/memmove_amd64.s

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
TEXT runtime·memmove(SB), NOSPLIT, $0-24
3232

3333
MOVQ to+0(FP), DI
34-
MOVQ fr+8(FP), SI
34+
MOVQ from+8(FP), SI
3535
MOVQ n+16(FP), BX
3636

3737
// REP instructions have a high startup cost, so we handle small sizes

‎src/pkg/runtime/mprof.goc

Lines changed: 1 addition & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package runtime
99
#include "runtime.h"
1010
#include "arch_GOARCH.h"
1111
#include "malloc.h"
12+
#include "mprof.h"
1213
#include "defs_GOOS_GOARCH.h"
1314
#include "type.h"
1415

@@ -20,58 +21,6 @@ static Lock proflock;
2021

2122
enum { MProf, BProf }; // profile types
2223

23-
// Per-call-stack profiling information.
24-
// Lookup by hashing call stack into a linked-list hash table.
25-
struct Bucket
26-
{
27-
Bucket *next; // next in hash list
28-
Bucket *allnext; // next in list of all mbuckets/bbuckets
29-
int32 typ;
30-
// Generally unions can break precise GC,
31-
// this one is fine because it does not contain pointers.
32-
union
33-
{
34-
struct // typ == MProf
35-
{
36-
// The following complex 3-stage scheme of stats accumulation
37-
// is required to obtain a consistent picture of mallocs and frees
38-
// for some point in time.
39-
// The problem is that mallocs come in real time, while frees
40-
// come only after a GC during concurrent sweeping. So if we would
41-
// naively count them, we would get a skew toward mallocs.
42-
//
43-
// Mallocs are accounted in recent stats.
44-
// Explicit frees are accounted in recent stats.
45-
// GC frees are accounted in prev stats.
46-
// After GC prev stats are added to final stats and
47-
// recent stats are moved into prev stats.
48-
uintptr allocs;
49-
uintptr frees;
50-
uintptr alloc_bytes;
51-
uintptr free_bytes;
52-
53-
uintptr prev_allocs; // since last but one till last gc
54-
uintptr prev_frees;
55-
uintptr prev_alloc_bytes;
56-
uintptr prev_free_bytes;
57-
58-
uintptr recent_allocs; // since last gc till now
59-
uintptr recent_frees;
60-
uintptr recent_alloc_bytes;
61-
uintptr recent_free_bytes;
62-
63-
};
64-
struct // typ == BProf
65-
{
66-
int64 count;
67-
int64 cycles;
68-
};
69-
};
70-
uintptr hash; // hash of size + stk
71-
uintptr size;
72-
uintptr nstk;
73-
uintptr stk[1];
74-
};
7524
enum {
7625
BuckHashSize = 179999,
7726
};

‎src/pkg/runtime/mprof.h

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright 2014 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// Per-call-stack profiling information.
6+
// Lookup by hashing call stack into a linked-list hash table.
7+
struct Bucket
8+
{
9+
Bucket *next; // next in hash list
10+
Bucket *allnext; // next in list of all mbuckets/bbuckets
11+
int32 typ;
12+
// Generally unions can break precise GC,
13+
// this one is fine because it does not contain pointers.
14+
union
15+
{
16+
struct // typ == MProf
17+
{
18+
// The following complex 3-stage scheme of stats accumulation
19+
// is required to obtain a consistent picture of mallocs and frees
20+
// for some point in time.
21+
// The problem is that mallocs come in real time, while frees
22+
// come only after a GC during concurrent sweeping. So if we would
23+
// naively count them, we would get a skew toward mallocs.
24+
//
25+
// Mallocs are accounted in recent stats.
26+
// Explicit frees are accounted in recent stats.
27+
// GC frees are accounted in prev stats.
28+
// After GC prev stats are added to final stats and
29+
// recent stats are moved into prev stats.
30+
uintptr allocs;
31+
uintptr frees;
32+
uintptr alloc_bytes;
33+
uintptr free_bytes;
34+
35+
uintptr prev_allocs; // since last but one till last gc
36+
uintptr prev_frees;
37+
uintptr prev_alloc_bytes;
38+
uintptr prev_free_bytes;
39+
40+
uintptr recent_allocs; // since last gc till now
41+
uintptr recent_frees;
42+
uintptr recent_alloc_bytes;
43+
uintptr recent_free_bytes;
44+
45+
};
46+
struct // typ == BProf
47+
{
48+
int64 count;
49+
int64 cycles;
50+
};
51+
};
52+
uintptr hash; // hash of size + stk
53+
uintptr size;
54+
uintptr nstk;
55+
uintptr stk[1];
56+
};

‎src/pkg/runtime/race.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ import (
1212
"unsafe"
1313
)
1414

15+
const (
16+
// TODO: where should these live?
17+
kindNoPointers = 1 << 7
18+
kindArray = 17
19+
kindStruct = 25
20+
)
21+
1522
// RaceDisable disables handling of race events in the current goroutine.
1623
func RaceDisable()
1724

@@ -32,3 +39,16 @@ func RaceSemrelease(s *uint32)
3239

3340
// private interface for the runtime
3441
const raceenabled = true
42+
43+
func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
44+
kind := t.kind &^ kindNoPointers
45+
if kind == kindArray || kind == kindStruct {
46+
// for composite objects we have to read every address
47+
// because a write might happen to any subobject.
48+
racereadrangepc(addr, int(t.size), callerpc, pc)
49+
} else {
50+
// for non-composite objects we can read just the start
51+
// address, as any write must write the first byte.
52+
racereadpc(addr, callerpc, pc)
53+
}
54+
}

‎src/pkg/runtime/race0.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,11 @@
88

99
package runtime
1010

11+
import (
12+
"unsafe"
13+
)
14+
1115
const raceenabled = false
16+
17+
func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
18+
}

‎src/pkg/runtime/string.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ func slicerunetostring(a []rune) string {
144144
}
145145

146146
type stringStruct struct {
147-
str *byte
147+
str unsafe.Pointer
148148
len int
149149
}
150150

@@ -156,7 +156,7 @@ func cstringToGo(str uintptr) (s string) {
156156
}
157157
}
158158
t := (*stringStruct)(unsafe.Pointer(&s))
159-
t.str = (*byte)(unsafe.Pointer(str))
159+
t.str = unsafe.Pointer(str)
160160
t.len = i
161161
return
162162
}

‎src/pkg/runtime/stubs.go

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ import "unsafe"
1111
// Assembly implementations are in various files, see comments with
1212
// each function.
1313

14+
const (
15+
ptrSize = unsafe.Sizeof((*byte)(nil))
16+
)
17+
1418
// rawstring allocates storage for a new string. The returned
1519
// string and byte slice both refer to the same storage.
1620
// The storage is not zeroed. Callers should use
@@ -26,5 +30,55 @@ func rawruneslice(size int) []rune
2630
//go:noescape
2731
func gogetcallerpc(p unsafe.Pointer) uintptr
2832

33+
//go:noescape
34+
func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
35+
36+
//go:noescape
37+
func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
38+
2939
//go:noescape
3040
func racereadrangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
41+
42+
// Should be a built-in for unsafe.Pointer?
43+
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
44+
return unsafe.Pointer(uintptr(p) + x)
45+
}
46+
47+
// Make a new object of the given type
48+
// in stubs.goc
49+
func unsafe_New(t *_type) unsafe.Pointer
50+
func unsafe_NewArray(t *_type, n uintptr) unsafe.Pointer
51+
52+
// memclr clears n bytes starting at ptr.
53+
// in memclr_*.s
54+
func memclr(ptr unsafe.Pointer, n uintptr)
55+
56+
// memmove copies n bytes from "from" to "to".
57+
// in memmove_*.s
58+
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
59+
60+
// in asm_*.s
61+
func fastrand2() uint32
62+
63+
// in asm_*.s
64+
// if *p == x { *p = y; return true } else { return false }, atomically
65+
//go:noescape
66+
func gocas(p *uint32, x uint32, y uint32) bool
67+
68+
// in asm_*.s
69+
//go:noescape
70+
func gohash(a *alg, p unsafe.Pointer, size uintptr, seed uintptr) uintptr
71+
72+
// in asm_*.s
73+
//go:noescape
74+
func goeq(alg *alg, p, q unsafe.Pointer, size uintptr) bool
75+
76+
// exported value for testing
77+
var hashLoad = loadFactor
78+
79+
// in asm_*.s
80+
//go:noescape
81+
func gomemeq(a, b unsafe.Pointer, size uintptr) bool
82+
83+
// Code pointer for the nohash algorithm. Used for producing better error messages.
84+
var nohashcode uintptr

‎src/pkg/runtime/stubs.goc

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,18 @@ func rawruneslice(size intgo) (b Slice) {
7575
func gostringW(str Slice) (s String) {
7676
s = runtime·gostringw((uint16*)str.array);
7777
}
78+
79+
#pragma textflag NOSPLIT
80+
func runtime·unsafe_New(t *Type) (ret *byte) {
81+
ret = runtime·cnew(t);
82+
}
83+
84+
#pragma textflag NOSPLIT
85+
func runtime·unsafe_NewArray(t *Type, n int) (ret *byte) {
86+
ret = runtime·cnewarray(t, n);
87+
}
88+
89+
#pragma textflag NOSPLIT
90+
func runtime·gocas(p *uint32, x uint32, y uint32) (ret bool) {
91+
ret = runtime·cas(p, x, y);
92+
}

0 commit comments

Comments
 (0)
Please sign in to comment.