Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

runtime: SIGILL: illegal instruction on linux/s390x #16126

Closed
mikioh opened this issue Jun 20, 2016 · 7 comments
Closed

runtime: SIGILL: illegal instruction on linux/s390x #16126

mikioh opened this issue Jun 20, 2016 · 7 comments
Labels
FrozenDueToAge
Milestone
Go1.8

Comments

@mikioh
Copy link
Contributor

mikioh commented Jun 20, 2016

See https://build.golang.org/log/d845753f4d6d7ecaffe9a99637d2f1017098bd81

ok      regexp/syntax   0.390s
SIGILL: illegal instruction
PC=0x54510 m=2

goroutine 0 [idle]:
runtime.retake(0x1064fb62f6bc3d, 0x0)
    /data/golang/workdir/go/src/runtime/proc.go:3709 +0x330 fp=0x3ffb75fee98 sp=0x3ffb75fee48
runtime.sysmon()
    /data/golang/workdir/go/src/runtime/proc.go:3622 +0x1ee fp=0x3ffb75fef10 sp=0x3ffb75fee98
runtime.mstart1()
    /data/golang/workdir/go/src/runtime/proc.go:1125 +0x128 fp=0x3ffb75fef30 sp=0x3ffb75fef10
runtime.mstart()
    /data/golang/workdir/go/src/runtime/proc.go:1095 +0x8e fp=0x3ffb75fef40 sp=0x3ffb75fef30

goroutine 1 [chan receive, locked to thread]:
runtime.gopark(0x297520, 0xc42042ef58, 0x27423a, 0xc, 0x170000000000003f, 0x3)
    /data/golang/workdir/go/src/runtime/proc.go:259 +0x18a fp=0xc4200499a0 sp=0xc420049978
runtime.goparkunlock(0xc42042ef58, 0x27423a, 0xc, 0x170000000037db40, 0x3)
    /data/golang/workdir/go/src/runtime/proc.go:265 +0x66 fp=0xc4200499d8 sp=0xc4200499a0
runtime.chanrecv(0x223000, 0xc42042ef00, 0x0, 0x10000c420049a98, 0x9b116)
    /data/golang/workdir/go/src/runtime/chan.go:496 +0x548 fp=0xc420049a60 sp=0xc4200499d8
runtime.chanrecv1(0x223000, 0xc42042ef00, 0x0)
    /data/golang/workdir/go/src/runtime/chan.go:378 +0x3a fp=0xc420049a90 sp=0xc420049a60
testing.(*T).Run(0xc42033a540, 0x275f9a, 0x11, 0x298228, 0x100000000044712)
    /data/golang/workdir/go/src/testing/testing.go:647 +0x3a0 fp=0xc420049b60 sp=0xc420049a90
testing.RunTests.func1(0xc42007c0c0)
    /data/golang/workdir/go/src/testing/testing.go:793 +0xe6 fp=0xc420049bf0 sp=0xc420049b60
testing.tRunner(0xc42007c0c0, 0xc420049d30)
    /data/golang/workdir/go/src/testing/testing.go:610 +0xd8 fp=0xc420049c28 sp=0xc420049bf0
testing.RunTests(0x296e98, 0x379d40, 0xc2, 0xc2, 0x1000000000ef037)
    /data/golang/workdir/go/src/testing/testing.go:799 +0x3da fp=0xc420049d50 sp=0xc420049c28
testing.(*M).Run(0xc420049ef8, 0x2ff0b0)
    /data/golang/workdir/go/src/testing/testing.go:743 +0x88 fp=0xc420049dd8 sp=0xc420049d50
runtime_test.TestMain(0xc420049ef8)
    /data/golang/workdir/go/src/runtime/crash_test.go:27 +0x2a fp=0xc420049e58 sp=0xc420049dd8
main.main()
    runtime/_test/_testmain.go:832 +0x14c fp=0xc420049f48 sp=0xc420049e58
runtime.main()
    /data/golang/workdir/go/src/runtime/proc.go:183 +0x296 fp=0xc420049f98 sp=0xc420049f48
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420049f98 sp=0xc420049f98

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420038fb0 sp=0xc420038fb0

goroutine 2 [force gc (idle)]:
runtime.gopark(0x297520, 0x37cbe0, 0x275400, 0xf, 0x1400000000000000, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:259 +0x18a fp=0xc420024750 sp=0xc420024728
runtime.goparkunlock(0x37cbe0, 0x275400, 0xf, 0x140000c420000340, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:265 +0x66 fp=0xc420024788 sp=0xc420024750
runtime.forcegchelper()
    /data/golang/workdir/go/src/runtime/proc.go:224 +0xba fp=0xc4200247b8 sp=0xc420024788
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc4200247b8 sp=0xc4200247b8
created by runtime.init.4
    /data/golang/workdir/go/src/runtime/proc.go:213 +0x30

goroutine 3 [GC sweep wait]:
runtime.gopark(0x297520, 0x37cf40, 0x274618, 0xd, 0x1400000000078cd0, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:259 +0x18a fp=0xc420024f40 sp=0xc420024f18
runtime.goparkunlock(0x37cf40, 0x274618, 0xd, 0x1400000000000000, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:265 +0x66 fp=0xc420024f78 sp=0xc420024f40
runtime.bgsweep(0xc420054000)
    /data/golang/workdir/go/src/runtime/mgcsweep.go:79 +0x154 fp=0xc420024fb0 sp=0xc420024f78
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420024fb0 sp=0xc420024fb0
created by runtime.gcenable
    /data/golang/workdir/go/src/runtime/mgc.go:195 +0x5a

goroutine 4 [finalizer wait]:
runtime.gopark(0x297520, 0x3a6f80, 0x274e06, 0xe, 0x1400000100000020, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:259 +0x18a fp=0xc420025710 sp=0xc4200256e8
runtime.goparkunlock(0x3a6f80, 0x274e06, 0xe, 0x1400002000000020, 0x1)
    /data/golang/workdir/go/src/runtime/proc.go:265 +0x66 fp=0xc420025748 sp=0xc420025710
runtime.runfinq()
    /data/golang/workdir/go/src/runtime/mfinal.go:158 +0xa4 fp=0xc4200257b8 sp=0xc420025748
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc4200257b8 sp=0xc4200257b8
created by runtime.createfing
    /data/golang/workdir/go/src/runtime/mfinal.go:139 +0x66

goroutine 5 [syscall]:
runtime.notetsleepg(0x37d098, 0xdbffe29f, 0x0)
    /data/golang/workdir/go/src/runtime/lock_futex.go:205 +0x4e fp=0xc420025f30 sp=0xc420025f08
runtime.timerproc()
    /data/golang/workdir/go/src/runtime/time.go:209 +0xec fp=0xc420025fb8 sp=0xc420025f30
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420025fb8 sp=0xc420025fb8
created by runtime.addtimerLocked
    /data/golang/workdir/go/src/runtime/time.go:116 +0x1be

goroutine 102943 [runnable]:
strconv.bigFtoa(0xc42033a068, 0x1, 0x44, 0x2, 0x660000001ffe4879, 0x10c6f7a0b5ed8e, 0xfffffffffffffff0, 0x360a20, 0x1ffffffffffffff, 0x160e54, ...)
    /data/golang/workdir/go/src/strconv/ftoa.go:155 fp=0xc420047528 sp=0xc420047528
strconv.genericFtoa(0xc42033a068, 0x1, 0x44, 0x3ef0c6f7a0b5ed8e, 0x66000000000249ca, 0x2, 0x40, 0x0, 0x0, 0x0)
    /data/golang/workdir/go/src/strconv/ftoa.go:149 +0x5de fp=0xc4200476f0 sp=0xc420047528
strconv.AppendFloat(0xc42033a068, 0x1, 0x44, 0x3ef0c6f7a0b5ed8e, 0x6600000000000000, 0x2, 0x40, 0x0, 0x0, 0x0)
    /data/golang/workdir/go/src/strconv/ftoa.go:51 +0x78 fp=0xc420047748 sp=0xc4200476f0
fmt.(*fmt).fmt_float(0xc42033a040, 0x3ef0c6f7a0b5ed8e, 0x40, 0x6620447928, 0x2)
    /data/golang/workdir/go/src/fmt/format.go:459 +0xba fp=0xc420047850 sp=0xc420047748
fmt.(*pp).fmtFloat(0xc42033a000, 0x3ef0c6f7a0b5ed8e, 0x40, 0x6620447928)
    /data/golang/workdir/go/src/fmt/print.go:395 +0x8e fp=0xc420047880 sp=0xc420047850
fmt.(*pp).printArg(0xc42033a000, 0x223800, 0xc4202959a8, 0x6600000001)
    /data/golang/workdir/go/src/fmt/print.go:630 +0x484 fp=0xc420047a28 sp=0xc420047880
fmt.(*pp).doPrintf(0xc42033a000, 0x271cb8, 0x5, 0xc420047d48, 0x1, 0x1)
    /data/golang/workdir/go/src/fmt/print.go:1078 +0x1636 fp=0xc420047ca8 sp=0xc420047a28
fmt.Sprintf(0x271cb8, 0x5, 0xc420047d48, 0x1, 0x1, 0x0, 0x0)
    /data/golang/workdir/go/src/fmt/print.go:196 +0x78 fp=0xc420047cf8 sp=0xc420047ca8
testing.fmtDuration(0x3e80, 0x0, 0x0)
    /data/golang/workdir/go/src/testing/testing.go:357 +0x152 fp=0xc420047d70 sp=0xc420047cf8
testing.(*T).report(0xc42033a540)
    /data/golang/workdir/go/src/testing/testing.go:760 +0x50 fp=0xc420047e98 sp=0xc420047d70
testing.tRunner.func1(0xc42033a540)
    /data/golang/workdir/go/src/testing/testing.go:601 +0x2b4 fp=0xc420047f50 sp=0xc420047e98
testing.tRunner(0xc42033a540, 0x298228)
    /data/golang/workdir/go/src/testing/testing.go:612 +0xe6 fp=0xc420047f88 sp=0xc420047f50
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420047f88 sp=0xc420047f88
created by testing.(*T).Run
    /data/golang/workdir/go/src/testing/testing.go:646 +0x376

goroutine 126 [GC worker (idle)]:
runtime.gopark(0x297340, 0xc4202a43d0, 0x2757e0, 0x10, 0x140000c420132f90, 0x0)
    /data/golang/workdir/go/src/runtime/proc.go:259 +0x18a fp=0xc420132f40 sp=0xc420132f18
runtime.gcBgMarkWorker(0xc420018000)
    /data/golang/workdir/go/src/runtime/mgc.go:1421 +0x104 fp=0xc420132fb0 sp=0xc420132f40
runtime.goexit()
    /data/golang/workdir/go/src/runtime/asm_s390x.s:1021 +0x2 fp=0xc420132fb0 sp=0xc420132fb0
created by runtime.gcBgMarkStartWorkers
    /data/golang/workdir/go/src/runtime/mgc.go:1342 +0x98

r0   0x0    r1   0xc42002a400
r2   0x989680   r3   0x3ab500
r4   0xc420274000   r5   0x2d5b1
r6   0x2    r7   0x1064fb62f6bc3d
r8   0x37d538   r9   0x3ffb75ff1f0
r10  0x14   r11  0x0
r12  0x2975e0   r13  0xc420000b60
r14  0x53dee    r15  0x3ffb75fee48
pc   0x54510    link 0x53dee
FAIL    runtime 8.677s
@ianlancetaylor ianlancetaylor added this to the Go1.7Maybe milestone Jun 20, 2016
@ianlancetaylor
Copy link
Contributor

CC @mundaym

@ianlancetaylor
Copy link
Contributor

54510 seems to be the start of the runqput function. This is the GNU binutils objdump output when I build the runtime test at that git revision:

0000000000054510 <runtime.runqput>:
   54510:       e3 30 d0 10 00 04       lg      %r3,16(%r13)
   54516:       ec 3f 00 cd a0 65       clgrjnl %r3,%r15,546b0 <runtime.runqput+0x1a0>
   5451c:       e3 ff 0f d0 ff 71       lay     %r15,-48(%r15,%r0)
   54522:       e3 e0 f0 00 00 24       stg     %r14,0(%r15)

Getting a SIGILL signal at that PC address does not make sense to me.

@mundaym mundaym mentioned this issue Jun 20, 2016
Closed
@mundaym
Copy link
Member

mundaym commented Jun 20, 2016

This has happened once before: #15441. I misread the problem as being in usleep() at the time.

objdump is pointing to a different place when I try it (perhaps due to external linking?). Nothing interesting there though. I'll try to recreate on the builder itself.

@mundaym
Copy link
Member

mundaym commented Jun 20, 2016

It looks like it was in the retake function. Still no obvious cause though.

   544f2:       e3 40 40 20 00 04       lg      %r4,32(%r4)
   544f8:       ec 40 fe aa 80 65       clgrj   %r4,%r0,8,5424c <runtime.retake+0x6c>
   544fe:       e3 10 d0 30 00 04       lg      %r1,48(%r13)
   54504:       ec 14 fe a4 80 65       clgrj   %r1,%r4,8,5424c <runtime.retake+0x6c>
   5450a:       e3 30 40 98 00 04       lg      %r3,152(%r4)
-> 54510:       ec 30 fe 9e 80 65       clgrj   %r3,%r0,8,5424c <runtime.retake+0x6c>
   54516:       e3 10 40 00 00 04       lg      %r1,0(%r4)
   5451c:       ec 13 fe 98 80 65       clgrj   %r1,%r3,8,5424c <runtime.retake+0x6c>
   54522:       92 01 30 e8             mvi     232(%r3),1
   54526:       e5 48 30 10 fa de       mvghi   16(%r3),-1314
   5452c:       b9 14 00 16             lgfr    %r1,%r6
   54530:       c2 18 00 00 00 01       agfi    %r1,1
   54536:       b9 14 00 61             lgfr    %r6,%r1
   5453a:       a7 f4 fe 67             j       54208 <runtime.retake+0x28>
   5453e:       e3 10 f0 38 00 04       lg      %r1,56(%r15)
   54544:       e3 10 f0 60 00 50       sty     %r1,96(%r15)
   5454a:       e3 e0 f0 00 00 04       lg      %r14,0(%r15)
   54550:       c2 f8 00 00 00 50       agfi    %r15,80
   54556:       07 fe                   br      %r14
   54558:       b9 04 00 5e             lgr     %r5,%r14
   5455c:       c0 e5 00 01 6f 5a       brasl   %r14,82410 <runtime.morestack_noctxt>
   54562:       a7 f4 fe 3f             j       541e0 <runtime.retake>

@ianlancetaylor
Copy link
Contributor

Moving to 1.8.

@ianlancetaylor ianlancetaylor modified the milestones: Go1.8, Go1.7Maybe Jun 28, 2016
@mikioh
Copy link
Contributor Author

mikioh commented Jun 30, 2016

A variation? https://build.golang.org/log/d35d1b9fe4ba87517e7071ecdee1fdef5e23eba0

ok      regexp/syntax   0.411s
signal: segmentation fault
FAIL    runtime 9.468s
2016/06/28 21:45:35 Failed: exit status 1

@mundaym
Copy link
Member

mundaym commented Aug 23, 2016

I am now convinced that these failures are the symptom of a kernel bug that was fixed by torvalds/linux@723cacb (backported to SLES 12 SP1 here).

I've updated the builder to 3.12.62-60.62-default #1 SMP Thu Aug 4 09:06:08 UTC 2016 (b0e5a26) and it now appears to be passing the runtime tests reliably (1000 runs, normally enough to see the problem many times).

I suggest closing this issue for now and reopening if we see the problem again.

@golang golang locked and limited conversation to collaborators Aug 23, 2017
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
FrozenDueToAge
Projects
None yet
Milestone
Go1.8
Development

No branches or pull requests
5 participants
@mikioh @ianlancetaylor @randall77 @gopherbot @mundaym