runtime: convert map implementation to Go.

It's a bit slower, but not painfully so. There is still room for improvement (saving space so we can use nosplit, and removing the requirement for hash/eq stubs). benchmark old ns/op new ns/op delta BenchmarkMegMap 23.5 24.2 +2.98% BenchmarkMegOneMap 14.9 15.7 +5.37% BenchmarkMegEqMap 71668 72234 +0.79% BenchmarkMegEmptyMap 4.05 4.93 +21.73% BenchmarkSmallStrMap 21.9 22.5 +2.74% BenchmarkMapStringKeysEight_16 23.1 26.3 +13.85% BenchmarkMapStringKeysEight_32 21.9 25.0 +14.16% BenchmarkMapStringKeysEight_64 21.9 25.1 +14.61% BenchmarkMapStringKeysEight_1M 21.9 25.0 +14.16% BenchmarkIntMap 21.8 12.5 -42.66% BenchmarkRepeatedLookupStrMapKey32 39.3 30.2 -23.16% BenchmarkRepeatedLookupStrMapKey1M 322353 322675 +0.10% BenchmarkNewEmptyMap 129 136 +5.43% BenchmarkMapIter 137 107 -21.90% BenchmarkMapIterEmpty 7.14 8.71 +21.99% BenchmarkSameLengthMap 5.24 6.82 +30.15% BenchmarkBigKeyMap 34.5 35.3 +2.32% BenchmarkBigValMap 36.1 36.1 +0.00% BenchmarkSmallKeyMap 26.9 26.7 -0.74% LGTM=rsc R=golang-codereviews, dave, dvyukov, rsc, gobot, khr CC=golang-codereviews https://golang.org/cl/99380043
golang · Jul 16, 2014 · 0c6b55e · 0c6b55e
1 parent 3b1b840
commit 0c6b55e
Show file tree

Hide file tree

Showing 29 changed files with 1,971 additions and 1,520 deletions.
diff --git a/src/cmd/api/goapi.go b/src/cmd/api/goapi.go
@@ -370,6 +370,14 @@ func (w *Walker) parseFile(dir, file string) (*ast.File, error) {
 			log.Fatalf("incorrect generated file: %s", err)
 		}
 	}
+	if w.context != nil && file == fmt.Sprintf("zruntime_defs_%s_%s.go", w.context.GOOS, w.context.GOARCH) {
+		// Just enough to keep the api checker happy.
+		src := "package runtime; type maptype struct{}; type _type struct{}; type alg struct{}"
+		f, err = parser.ParseFile(fset, filename, src, 0)
+		if err != nil {
+			log.Fatalf("incorrect generated file: %s", err)
+		}
+	}
 
 	if f == nil {
 		f, err = parser.ParseFile(fset, filename, nil, 0)
@@ -488,6 +496,11 @@ func (w *Walker) Import(name string) (pkg *types.Package) {
 		if !contains(filenames, n) {
 			filenames = append(filenames, n)
 		}
+
+		n = fmt.Sprintf("zruntime_defs_%s_%s.go", w.context.GOOS, w.context.GOARCH)
+		if !contains(filenames, n) {
+			filenames = append(filenames, n)
+		}
 	}
 
 	// Parse package files.

diff --git a/src/cmd/dist/buildruntime.c b/src/cmd/dist/buildruntime.c
@@ -231,6 +231,8 @@ mkzasm(char *dir, char *file)
 				aggr = "cbctxt";
 			else if(streq(fields.p[1], "SEH"))
 				aggr = "seh";
+			else if(streq(fields.p[1], "Alg"))
+				aggr = "alg";
 		}
 		if(hasprefix(lines.p[i], "}"))
 			aggr = nil;

diff --git a/src/cmd/ld/dwarf.c b/src/cmd/ld/dwarf.c
@@ -1222,7 +1222,7 @@ synthesizemaptypes(DWDie *die)
 	DWAttr *a;
 
 	hash		= walktypedef(defgotype(lookup_or_diag("type.runtime.hmap")));
-	bucket		= walktypedef(defgotype(lookup_or_diag("type.runtime.bucket")));
+	bucket		= walktypedef(defgotype(lookup_or_diag("type.runtime.bmap")));
 
 	if (hash == nil)
 		return;

diff --git a/src/pkg/reflect/asm_386.s b/src/pkg/reflect/asm_386.s
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
 	MOVL	CX, 4(SP)
 	CALL	·callMethod(SB)
 	RET
+
+// Stubs to give reflect package access to runtime services
+// TODO: should probably be done another way.
+TEXT ·makemap(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_makemap(SB)
+TEXT ·mapaccess(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapaccess(SB)
+TEXT ·mapassign(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapassign(SB)
+TEXT ·mapdelete(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapdelete(SB)
+TEXT ·mapiterinit(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterinit(SB)
+TEXT ·mapiterkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterkey(SB)
+TEXT ·mapiternext(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiternext(SB)
+TEXT ·maplen(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_maplen(SB)
+TEXT ·ismapkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_ismapkey(SB)
diff --git a/src/pkg/reflect/asm_amd64.s b/src/pkg/reflect/asm_amd64.s
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16
 	MOVQ	CX, 8(SP)
 	CALL	·callMethod(SB)
 	RET
+
+// Stubs to give reflect package access to runtime services
+// TODO: should probably be done another way.
+TEXT ·makemap(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_makemap(SB)
+TEXT ·mapaccess(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapaccess(SB)
+TEXT ·mapassign(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapassign(SB)
+TEXT ·mapdelete(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapdelete(SB)
+TEXT ·mapiterinit(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterinit(SB)
+TEXT ·mapiterkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterkey(SB)
+TEXT ·mapiternext(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiternext(SB)
+TEXT ·maplen(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_maplen(SB)
+TEXT ·ismapkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_ismapkey(SB)
diff --git a/src/pkg/reflect/asm_amd64p32.s b/src/pkg/reflect/asm_amd64p32.s
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
 	MOVL	CX, 4(SP)
 	CALL	·callMethod(SB)
 	RET
+
+// Stubs to give reflect package access to runtime services
+// TODO: should probably be done another way.
+TEXT ·makemap(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_makemap(SB)
+TEXT ·mapaccess(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapaccess(SB)
+TEXT ·mapassign(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapassign(SB)
+TEXT ·mapdelete(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapdelete(SB)
+TEXT ·mapiterinit(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterinit(SB)
+TEXT ·mapiterkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiterkey(SB)
+TEXT ·mapiternext(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_mapiternext(SB)
+TEXT ·maplen(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_maplen(SB)
+TEXT ·ismapkey(SB),NOSPLIT,$0-0
+	JMP	runtime·reflect_ismapkey(SB)
diff --git a/src/pkg/reflect/asm_arm.s b/src/pkg/reflect/asm_arm.s
@@ -25,3 +25,24 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8
 	MOVW	R1, 8(R13)
 	BL	·callMethod(SB)
 	RET
+
+// Stubs to give reflect package access to runtime services
+// TODO: should probably be done another way.
+TEXT ·makemap(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_makemap(SB)
+TEXT ·mapaccess(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapaccess(SB)
+TEXT ·mapassign(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapassign(SB)
+TEXT ·mapdelete(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapdelete(SB)
+TEXT ·mapiterinit(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapiterinit(SB)
+TEXT ·mapiterkey(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapiterkey(SB)
+TEXT ·mapiternext(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_mapiternext(SB)
+TEXT ·maplen(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_maplen(SB)
+TEXT ·ismapkey(SB),NOSPLIT,$-4-0
+	B	runtime·reflect_ismapkey(SB)
diff --git a/src/pkg/runtime/alg.goc b/src/pkg/runtime/alg.goc
@@ -425,6 +425,8 @@ runtime·nohash(uintptr *h, uintptr s, void *a)
 	runtime·panicstring("hash of unhashable type");
 }
 
+extern uintptr runtime·nohashcode;
+
 void
 runtime·noequal(bool *eq, uintptr s, void *a, void *b)
 {
@@ -471,6 +473,7 @@ byte runtime·aeskeysched[HashRandomBytes];
 void
 runtime·hashinit(void)
 {
+	runtime·nohashcode = (uintptr)runtime·nohash;
         if(NaCl)
                 return;
 

diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s
@@ -1107,6 +1107,14 @@ TEXT runtime·memeq(SB),NOSPLIT,$0-12
 	MOVL	count+8(FP), BX
 	JMP	runtime·memeqbody(SB)
 
+TEXT runtime·gomemeq(SB),NOSPLIT,$0-13
+	MOVL	a+0(FP), SI
+	MOVL	b+4(FP), DI
+	MOVL	size+8(FP), BX
+	CALL	runtime·memeqbody(SB)
+	MOVB	AX, ret+12(FP)
+	RET
+
 // eqstring tests whether two strings are equal.
 // See runtime_test.go:eqstring_generic for
 // equivlaent Go code.
@@ -2197,3 +2205,81 @@ TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
 
 TEXT runtime·timenow(SB), NOSPLIT, $0-0
 	JMP	time·now(SB)
+
+TEXT runtime·fastrand2(SB), NOSPLIT, $0-4
+	get_tls(CX)
+	MOVL	g(CX), AX
+	MOVL	g_m(AX), AX
+	MOVL	m_fastrand(AX), DX
+	ADDL	DX, DX
+	MOVL	DX, BX
+	XORL	$0x88888eef, DX
+	CMOVLMI	BX, DX
+	MOVL	DX, m_fastrand(AX)
+	MOVL	DX, ret+0(FP)
+	RET
+
+// The gohash and goeq trampolines are necessary while we have
+// both Go and C calls to alg functions.  Once we move all call
+// sites to Go, we can redo the hash/eq functions to use the
+// Go calling convention and remove these.
+
+// convert call to:
+//   func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
+// to:
+//   func (hash *uintptr, size uintptr, p unsafe.Pointer)
+TEXT runtime·gohash(SB), NOSPLIT, $12-20
+	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
+	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
+	MOVL	a+0(FP), AX
+	MOVL	alg_hash(AX), AX
+	MOVL	p+4(FP), CX
+	MOVL	size+8(FP), DX
+	MOVL	seed+12(FP), DI
+	MOVL	DI, ret+16(FP)
+	LEAL	ret+16(FP), SI
+	MOVL	SI, 0(SP)
+	MOVL	DX, 4(SP)
+	MOVL	CX, 8(SP)
+	PCDATA  $PCDATA_StackMapIndex, $0
+	CALL	*AX
+	RET
+
+DATA gcargs_gohash<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gcargs_gohash<>+0x04(SB)/4, $10  // 5 args
+DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
+GLOBL gcargs_gohash<>(SB),RODATA,$12
+
+DATA gclocals_gohash<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gclocals_gohash<>+0x04(SB)/4, $0  // 0 locals
+GLOBL gclocals_gohash<>(SB),RODATA,$8
+
+// convert call to:
+//   func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
+// to:
+//   func (eq *bool, size uintptr, p, q unsafe.Pointer)
+TEXT runtime·goeq(SB), NOSPLIT, $16-17
+	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
+	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
+	MOVL	alg+0(FP), AX
+	MOVL	alg_equal(AX), AX
+	MOVL	p+4(FP), CX
+	MOVL	q+8(FP), DX
+	MOVL	size+12(FP), DI
+	LEAL	ret+16(FP), SI
+	MOVL	SI, 0(SP)
+	MOVL	DI, 4(SP)
+	MOVL	CX, 8(SP)
+	MOVL	DX, 12(SP)
+	PCDATA  $PCDATA_StackMapIndex, $0
+	CALL	*AX
+	RET
+
+DATA gcargs_goeq<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gcargs_goeq<>+0x04(SB)/4, $10  // 5 args
+DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
+GLOBL gcargs_goeq<>(SB),RODATA,$12
+
+DATA gclocals_goeq<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gclocals_goeq<>+0x04(SB)/4, $0  // 0 locals
+GLOBL gclocals_goeq<>(SB),RODATA,$8
diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s
@@ -1075,6 +1075,14 @@ TEXT runtime·memeq(SB),NOSPLIT,$0-24
 	MOVQ	count+16(FP), BX
 	JMP	runtime·memeqbody(SB)
 
+TEXT runtime·gomemeq(SB),NOSPLIT,$0-25
+	MOVQ	a+0(FP), SI
+	MOVQ	b+8(FP), DI
+	MOVQ	size+16(FP), BX
+	CALL	runtime·memeqbody(SB)
+	MOVB	AX, ret+24(FP)
+	RET
+
 // eqstring tests whether two strings are equal.
 // See runtime_test.go:eqstring_generic for
 // equivlaent Go code.
@@ -2236,3 +2244,81 @@ TEXT runtime·duffcopy(SB), NOSPLIT, $0-0
 
 TEXT runtime·timenow(SB), NOSPLIT, $0-0
 	JMP	time·now(SB)
+
+TEXT runtime·fastrand2(SB), NOSPLIT, $0-4
+	get_tls(CX)
+	MOVQ	g(CX), AX
+	MOVQ	g_m(AX), AX
+	MOVL	m_fastrand(AX), DX
+	ADDL	DX, DX
+	MOVL	DX, BX
+	XORL	$0x88888eef, DX
+	CMOVLMI	BX, DX
+	MOVL	DX, m_fastrand(AX)
+	MOVL	DX, ret+0(FP)
+	RET
+
+// The gohash and goeq trampolines are necessary while we have
+// both Go and C calls to alg functions.  Once we move all call
+// sites to Go, we can redo the hash/eq functions to use the
+// Go calling convention and remove these.
+
+// convert call to:
+//   func (alg unsafe.Pointer, p unsafe.Pointer, size uintpr, seed uintptr) uintptr
+// to:
+//   func (hash *uintptr, size uintptr, p unsafe.Pointer)
+TEXT runtime·gohash(SB), NOSPLIT, $24-40
+	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_gohash<>(SB)
+	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_gohash<>(SB)
+	MOVQ	a+0(FP), AX
+	MOVQ	alg_hash(AX), AX
+	MOVQ	p+8(FP), CX
+	MOVQ	size+16(FP), DX
+	MOVQ	seed+24(FP), DI
+	MOVQ	DI, ret+32(FP)
+	LEAQ	ret+32(FP), SI	// TODO: go vet complains here: "invalid LEAQ of ret+32(FP); bool is 1-byte value"
+	MOVQ	SI, 0(SP)
+	MOVQ	DX, 8(SP)
+	MOVQ	CX, 16(SP)
+	PCDATA  $PCDATA_StackMapIndex, $0
+	CALL	*AX
+	RET
+
+DATA gcargs_gohash<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gcargs_gohash<>+0x04(SB)/4, $10  // 5 args
+DATA gcargs_gohash<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2))
+GLOBL gcargs_gohash<>(SB),RODATA,$12
+
+DATA gclocals_gohash<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gclocals_gohash<>+0x04(SB)/4, $0  // 0 locals
+GLOBL gclocals_gohash<>(SB),RODATA,$8
+
+// convert call to:
+//   func (alg unsafe.Pointer, p, q unsafe.Pointer, size uintptr) bool
+// to:
+//   func (eq *bool, size uintptr, p, q unsafe.Pointer)
+TEXT runtime·goeq(SB), NOSPLIT, $32-33
+	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_goeq<>(SB)
+	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_goeq<>(SB)
+	MOVQ	alg+0(FP), AX
+	MOVQ	alg_equal(AX), AX
+	MOVQ	p+8(FP), CX
+	MOVQ	q+16(FP), DX
+	MOVQ	size+24(FP), DI
+	LEAQ	ret+32(FP), SI
+	MOVQ	SI, 0(SP)
+	MOVQ	DI, 8(SP)
+	MOVQ	CX, 16(SP)
+	MOVQ	DX, 24(SP)
+	PCDATA  $PCDATA_StackMapIndex, $0
+	CALL	*AX
+	RET
+
+DATA gcargs_goeq<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gcargs_goeq<>+0x04(SB)/4, $10  // 5 args
+DATA gcargs_goeq<>+0x08(SB)/4, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsPointer<<4))
+GLOBL gcargs_goeq<>(SB),RODATA,$12
+
+DATA gclocals_goeq<>+0x00(SB)/4, $1  // 1 stackmap
+DATA gclocals_goeq<>+0x04(SB)/4, $0  // 0 locals
+GLOBL gclocals_goeq<>(SB),RODATA,$8