Skip to content

x/net/idna: ToASCII bug causes memory and cpu leak #22184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom
Closed
ma6174 opened this issue Oct 9, 2017 · 10 comments
Closed

x/net/idna: ToASCII bug causes memory and cpu leak #22184

ma6174 opened this issue Oct 9, 2017 · 10 comments
Labels
FrozenDueToAge NeedsFix The path to resolution is known, but the work has not been done.
Milestone
Go1.10

Comments

@ma6174
Copy link

ma6174 commented Oct 9, 2017

What version of Go are you using (go version)?

go version go1.9.1 darwin/amd64

Does this issue reproduce with the latest release?

Yes

What operating system and processor architecture are you using (go env)?

GOARCH="amd64"
GOBIN=""
GOEXE=""
GOHOSTARCH="amd64"
GOHOSTOS="darwin"
GOOS="darwin"
GOPATH="/Users/ma6174/gopath"
GORACE=""
GOROOT="/Users/ma6174/go1.9.1"
GOTOOLDIR="/Users/ma6174/go1.9.1/pkg/tool/darwin_amd64"
GCCGO="gccgo"
CC="clang"
GOGCCFLAGS="-fPIC -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/zz/wrwm9_1n7rq094rkczx_pm280000gn/T/go-build805739314=/tmp/go-build -gno-record-gcc-switches -fno-common"
CXX="clang++"
CGO_ENABLED="1"
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"

What did you do?

package main

import (
	"fmt"
	"net/http"
	"runtime"
)

func main() {
	fmt.Println("start", runtime.Version(), runtime.GOOS, runtime.GOARCH)
	u := "http://\xed/"
	_, err := http.Get(u)
	fmt.Printf("==> %v", err)
}

What did you expect to see?

program exit with an error of no such host or invalid domain name

in go1.8

start go1.8.1 linux amd64
==> Get http://%ED/: dial tcp: lookup xn--zn7c on 192.168.1.1:53: no such host

in go1.7 or before

start go1.7.5 linux amd64
==> Get http://%ED/: dial tcp: lookup �: invalid domain name

What did you see instead?

program fataled until oom, and more than 100% cpu usage.

start go1.9.1 linux amd64
fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x66cb10, 0x16)
	/Users/ma6174/go1.9.1/src/runtime/panic.go:605 +0x95
runtime.sysMap(0xc4247c0000, 0x2970000, 0xc42050a400, 0x7d2df8)
	/Users/ma6174/go1.9.1/src/runtime/mem_linux.go:216 +0x1d0
runtime.(*mheap).sysAlloc(0x7b9da0, 0x2970000, 0xc4204f3e20)
	/Users/ma6174/go1.9.1/src/runtime/malloc.go:470 +0xd7
runtime.(*mheap).grow(0x7b9da0, 0x14b2, 0x0)
	/Users/ma6174/go1.9.1/src/runtime/mheap.go:887 +0x60
runtime.(*mheap).allocSpanLocked(0x7b9da0, 0x14b2, 0x7d2e08, 0xc4204f3eb0)
	/Users/ma6174/go1.9.1/src/runtime/mheap.go:800 +0x334
runtime.(*mheap).alloc_m(0x7b9da0, 0x14b2, 0xffffffffffff0101, 0xc42050a480)
	/Users/ma6174/go1.9.1/src/runtime/mheap.go:666 +0x118
runtime.(*mheap).alloc.func1()
	/Users/ma6174/go1.9.1/src/runtime/mheap.go:733 +0x4d
runtime.systemstack(0xc4204f3f08)
	/Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:360 +0xab
runtime.(*mheap).alloc(0x7b9da0, 0x14b2, 0x101, 0xc420000180)
	/Users/ma6174/go1.9.1/src/runtime/mheap.go:732 +0xa1
runtime.largeAlloc(0x2964000, 0x450100, 0x10000c420001d28)
	/Users/ma6174/go1.9.1/src/runtime/malloc.go:827 +0x98
runtime.mallocgc.func1()
	/Users/ma6174/go1.9.1/src/runtime/malloc.go:722 +0x46
runtime.systemstack(0xc42050a4b8)
	/Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:344 +0x79
runtime.mstart()
	/Users/ma6174/go1.9.1/src/runtime/proc.go:1125

goroutine 1 [running]:
runtime.systemstack_switch()
	/Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:298 fp=0xc4200593a8 sp=0xc4200593a0 pc=0x454370
runtime.mallocgc(0x2964000, 0x0, 0x1a7c000, 0xc421fac000)
	/Users/ma6174/go1.9.1/src/runtime/malloc.go:721 +0x7b8 fp=0xc420059450 sp=0xc4200593a8 pc=0x410608
runtime.growslice(0x610840, 0xc421fac000, 0x211bfff, 0x211c000, 0x211c002, 0xc421fac000, 0x1a7c000, 0x211c000)
	/Users/ma6174/go1.9.1/src/runtime/slice.go:140 +0x20c fp=0xc4200594b8 sp=0xc420059450 pc=0x44057c
vendor/golang_org/x/net/idna.validateAndMap(0x7a9720, 0x6693b8, 0x1, 0x7f89364e4728, 0x0, 0x30, 0x7b2880)
	/Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:423 +0x480 fp=0xc420059580 sp=0xc4200594b8 pc=0x59e0b0
vendor/golang_org/x/net/idna.(*Profile).process(0x7a9720, 0x6693b8, 0x1, 0x580001, 0x6693b8, 0x1, 0xc42008ec3a, 0xffffffffffffffff)
	/Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:293 +0xb0a fp=0xc420059680 sp=0xc420059580 pc=0x59dbda
vendor/golang_org/x/net/idna.(*Profile).ToASCII(0x7a9720, 0x6693b8, 0x1, 0x6693b8, 0x1, 0x6693b8, 0x1)
	/Users/ma6174/go1.9.1/src/vendor/golang_org/x/net/idna/idna.go:189 +0x44 fp=0xc4200596d0 sp=0xc420059680 pc=0x59cdb4
net/http.idnaASCII(0x6693b8, 0x1, 0x1, 0x580028, 0x6693b8, 0x1)
	/Users/ma6174/go1.9.1/src/net/http/request.go:650 +0x96 fp=0xc420059718 sp=0xc4200596d0 pc=0x5c65c6
net/http.canonicalAddr(0xc420116000, 0x0, 0x1)
	/Users/ma6174/go1.9.1/src/net/http/transport.go:2081 +0x51 fp=0xc420059768 sp=0xc420059718 pc=0x5d8ab1
net/http.(*Transport).connectMethodForRequest(0x7ae100, 0xc42008ed20, 0x0, 0x6693b1, 0x4, 0x0, 0x0, 0x100, 0xc0)
	/Users/ma6174/go1.9.1/src/net/http/transport.go:614 +0xe8 fp=0xc420059820 sp=0xc420059768 pc=0x5cf5f8
net/http.(*Transport).RoundTrip(0x7ae100, 0xc420118000, 0x7ae100, 0x0, 0x0)
	/Users/ma6174/go1.9.1/src/net/http/transport.go:390 +0x5dd fp=0xc420059a88 sp=0xc420059820 pc=0x5ce19d
net/http.send(0xc420118000, 0x77c9c0, 0x7ae100, 0x0, 0x0, 0x0, 0xc42000c058, 0x100, 0xc420059c58, 0x1)
	/Users/ma6174/go1.9.1/src/net/http/client.go:249 +0x1a9 fp=0xc420059be8 sp=0xc420059a88 pc=0x5a5fd9
net/http.(*Client).send(0x7b1c00, 0xc420118000, 0x0, 0x0, 0x0, 0xc42000c058, 0x0, 0x1, 0xc420059ce0)
	/Users/ma6174/go1.9.1/src/net/http/client.go:173 +0xfd fp=0xc420059c68 sp=0xc420059be8 pc=0x5a5c6d
net/http.(*Client).Do(0x7b1c00, 0xc420118000, 0x6693b1, 0x9, 0x0)
	/Users/ma6174/go1.9.1/src/net/http/client.go:602 +0x28d fp=0xc420059e38 sp=0xc420059c68 pc=0x5a737d
net/http.(*Client).Get(0x7b1c00, 0x6693b1, 0x9, 0x1a, 0x0, 0x0)
	/Users/ma6174/go1.9.1/src/net/http/client.go:393 +0xa7 fp=0xc420059e90 sp=0xc420059e38 pc=0x5a6e97
net/http.Get(0x6693b1, 0x9, 0x4, 0x1a, 0x0)
	/Users/ma6174/go1.9.1/src/net/http/client.go:367 +0x41 fp=0xc420059ed0 sp=0xc420059e90 pc=0x5a6dc1
main.main()
	/Users/ma6174/url_get.go:12 +0x10c fp=0xc420059f80 sp=0xc420059ed0 pc=0x5e715c
runtime.main()
	/Users/ma6174/go1.9.1/src/runtime/proc.go:185 +0x20d fp=0xc420059fe0 sp=0xc420059f80 pc=0x42b11d
runtime.goexit()
	/Users/ma6174/go1.9.1/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420059fe8 sp=0xc420059fe0 pc=0x456ef1

the problem code is:

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/tables.go#L484-L487

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/idna.go#L401-L404

https://github.com/golang/go/blob/release-branch.go1.9/src/vendor/golang_org/x/net/idna/idna.go#L423

len(s) == 1 and lookupString() return 0, 0;
in validateAndMap, sz=0, i+=0, i not changed, so the for loop cannot break, this is a cpu leak;
v=0 hit select unknown branch, call b = append(b, "\ufffd"...), this is a memory leak.
@odeke-em odeke-em changed the title vendor/golang_org/x/net/idna: ToASCII bug cause memory and cpu leak x/net/idna: ToASCII bug causes memory and cpu leak Oct 9, 2017
@odeke-em
Copy link
Member

odeke-em commented Oct 9, 2017

/cc @mpvl

@ghost
Copy link

ghost commented Oct 10, 2017 •
edited by ghost
Loading

Reproduces on linux/amd64 (x/net/idna master).

package idna

import (
	"testing"

	"golang.org/x/net/idna"
)

func TestToASCII(t *testing.T) {
	idna.Lookup.ToASCII("\xed")
}

@SunRunAway
Copy link

any updates here?

@ALTree ALTree added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label Oct 25, 2017
@ALTree ALTree added this to the Unreleased milestone Oct 25, 2017
@ALTree ALTree added NeedsFix The path to resolution is known, but the work has not been done. and removed NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. labels Oct 25, 2017
@ALTree ALTree modified the milestones: Unreleased, Go1.10 Oct 25, 2017
@ALTree
Copy link
Member

ALTree commented Oct 25, 2017

So this is an issue in the x/net/idna master; setting the milestone to 1.10 since the faulty idna version is also currently vendored in the 1.9 tree.

@corona10
Copy link
Contributor

@ALTree Can I take a look this issue?

@ALTree
Copy link
Member

ALTree commented Oct 26, 2017 •
edited
Loading

@corona10 sure! I'm not working on this.

@gopherbot
Copy link
Contributor

Change https://golang.org/cl/73610 mentions this issue: idna: Fix ToASCII bug causes memory and cpu leak

@corona10
Copy link
Contributor

@ALTree I submitted the patch PTAL!

@gopherbot
Copy link
Contributor

Change https://golang.org/cl/73630 mentions this issue: idna: Fix ToASCII bug causes memory and cpu leak

@gopherbot
Copy link
Contributor

Change https://golang.org/cl/73730 mentions this issue: idna: Fix functions can causes memory and cpu leak.

@golang golang locked and limited conversation to collaborators Nov 1, 2018
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
FrozenDueToAge NeedsFix The path to resolution is known, but the work has not been done.
Projects
None yet
Milestone
Go1.10
Development

No branches or pull requests
6 participants
@SunRunAway @ma6174 @ALTree @odeke-em @corona10 @gopherbot