Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

os: occasional TestProgWideChdir deadlocks on arm builders #32330

Open
bcmills opened this issue May 30, 2019 · 2 comments
Open

os: occasional TestProgWideChdir deadlocks on arm builders #32330

bcmills opened this issue May 30, 2019 · 2 comments
Labels
NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. Testing An issue that has been verified to require only test changes, not just a test failure.
Milestone
Backlog

Comments

@bcmills
Copy link
Contributor

bcmills commented May 30, 2019

Possibly the same root cause as #29633 (CC @ianlancetaylor).
Two different failure modes, but the same test on the stack both times.

https://build.golang.org/log/e01b6ffb1a8fd323903ae672f43a6b086b8d217f

SIGQUIT: quit
PC=0x6c8c8 m=3 sigcode=0

goroutine 6 [syscall]:
runtime.notetsleepg(0x2b6408, 0xffffffff, 0xffffffff, 0x1)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/runtime/lock_sema.go:286 +0x24 fp=0x42cfc8 sp=0x42cfa4 pc=0x19ec0
os/signal.signal_recv(0x0)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/runtime/sigqueue.go:139 +0x130 fp=0x42cfe0 sp=0x42cfc8 pc=0x54894
os/signal.loop()
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/signal/signal_unix.go:23 +0x14 fp=0x42cfec sp=0x42cfe0 pc=0x114cfc
runtime.goexit()
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/runtime/asm_arm.s:868 +0x4 fp=0x42cfec sp=0x42cfec pc=0x6bad4
created by os/signal.init.0
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/signal/signal_unix.go:29 +0x30

goroutine 1 [chan receive]:
testing.(*T).Run(0x47fae0, 0x18e070, 0x11, 0x199074, 0x1)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:961 +0x2cc
testing.runTests.func1(0x47e000)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:1207 +0x68
testing.tRunner(0x47e000, 0x42a710)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:909 +0xa0
testing.runTests(0x40c090, 0x2a4a50, 0x7b, 0x7b, 0x0)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:1205 +0x238
testing.(*M).Run(0x440100, 0x0)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:1122 +0x130
main.main()
	_testmain.go:318 +0x120

goroutine 51 [runnable]:
os_test.TestProgWideChdir(0x47fae0)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1317 +0x35c
testing.tRunner(0x47fae0, 0x199074)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:909 +0xa0
created by testing.(*T).Run
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/testing/testing.go:960 +0x2ac

goroutine 56 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x4)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

goroutine 57 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x5)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

goroutine 58 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x6)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

goroutine 59 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x7)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

goroutine 60 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x8)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

goroutine 61 [runnable]:
runtime.LockOSThread()
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/runtime/proc.go:3530 +0x5c
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x9)
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1281 +0x124
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-46fd677069df/go/src/os/os_test.go:1272 +0x80

trap    0x0
error   0x0
oldmask 0x0
r0      0x58
r1      0x3
r2      0x0
r3      0x0
r4      0x42e58c
r5      0x1
r6      0x0
r7      0x199300
r8      0x7
r9      0x1
r10     0x400e00
fp      0xffffffc0
ip      0x5e
sp      0x42cf50
lr      0x3b618
pc      0x6c8c8
cpsr    0x40000010
fault   0x0
*** Test killed with quit: ran too long (37m0s).
FAIL	os	2220.201s

https://build.golang.org/log/7b59d0589251c325136e98f4b76193ae919399b4

SIGQUIT: quit
PC=0x6c974 m=5 sigcode=0

goroutine 0 [idle]:
runtime: unexpected return pc for runtime.kevent called from 0x40
stack: frame={sp:0x4d96f4, fp:0x4d96fc} stack=[0x4d8000,0x4da000)
004d9674:  00000000  00000000  00000000  00000000 
004d9684:  00000000  00000000  00000000  00000000 
004d9694:  00000000  00000000  00000000  00000000 
004d96a4:  00000000  00000000  00000000  00000000 
004d96b4:  00000000  00000000  00000000  00000000 
004d96c4:  00000000  00000000  00000000  00000000 
004d96d4:  00000000  00000000  00000000  00000000 
004d96e4:  00000000  00000000  00000000  0003b294 <runtime.netpoll+376> 
004d96f4: <00000040  00000000 >000448c4 <runtime.findrunnable+960>  00000005 
004d9704:  00000000  00000000  004d973c  00000040 
004d9714:  00000000  00000000  00000000  00000000 
004d9724:  00000000  00000000  00000000  00000000 
004d9734:  00000000  00000000  00000000  00000000 
004d9744:  00000000  00000000  00000000  00000000 
004d9754:  00000000  00000000  00000000  00000000 
004d9764:  00000000  00000000  00000000  00000000 
004d9774:  00000000  00000000 
runtime.kevent(0x5, 0x0, 0x0, 0x4d973c, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/runtime/sys_openbsd_arm.s:357 +0x30

goroutine 1 [chan receive]:
testing.(*T).Run(0x47fae0, 0x18e070, 0x11, 0x199074, 0x1)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:961 +0x2cc
testing.runTests.func1(0x47e000)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:1207 +0x68
testing.tRunner(0x47e000, 0x42a710)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:909 +0xa0
testing.runTests(0x40c090, 0x2a4a50, 0x7b, 0x7b, 0x0)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:1205 +0x238
testing.(*M).Run(0x440100, 0x0)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:1122 +0x130
main.main()
	_testmain.go:318 +0x120

goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/runtime/sigqueue.go:139 +0x130
os/signal.loop()
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/signal/signal_unix.go:23 +0x14
created by os/signal.init.0
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/signal/signal_unix.go:29 +0x30

goroutine 51 [runnable]:
os_test.TestProgWideChdir(0x47fae0)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1317 +0x35c
testing.tRunner(0x47fae0, 0x199074)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:909 +0xa0
created by testing.(*T).Run
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/testing/testing.go:960 +0x2ac

goroutine 56 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x4)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

goroutine 57 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x5)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

goroutine 58 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x6)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

goroutine 59 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x7)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

goroutine 60 [runnable]:
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x8)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

goroutine 61 [runnable]:
runtime.LockOSThread()
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/runtime/proc.go:3530 +0x5c
os_test.TestProgWideChdir.func1(0x50e880, 0x47fae0, 0x50e8c0, 0x9)
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1281 +0x124
created by os_test.TestProgWideChdir
	/home/gopher/build/openbsd-arm-c468ad04177c/go/src/os/os_test.go:1272 +0x80

trap    0x0
error   0x0
oldmask 0x0
r0      0x4
r1      0x0
r2      0x0
r3      0x4d973c
r4      0x40
r5      0x0
r6      0x0
r7      0x1993c0
r8      0x7
r9      0x2
r10     0x4015e0
fp      0x29004c
ip      0x48
sp      0x4d96f4
lr      0x3b294
pc      0x6c974
cpsr    0x20000010
fault   0x0
*** Test killed with quit: ran too long (37m0s).
FAIL	os	2220.194s

@bcmills bcmills added Testing An issue that has been verified to require only test changes, not just a test failure. OS-OpenBSD labels May 30, 2019
@bcmills bcmills added this to the Go1.13 milestone May 30, 2019
@bcmills
Copy link
Contributor Author

bcmills commented May 30, 2019

CC @4a6f656c

@bcmills bcmills added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label May 30, 2019
@andybons andybons modified the milestones: Go1.13, Go1.14 Jul 8, 2019
@bcmills
Copy link
Contributor Author

bcmills commented Jul 31, 2019

This looks like the same failure on linux-arm. Might not be an OpenBSD-specific issue at all.

https://build.golang.org/log/1838d40bb2d1457d52c6d81bfa96e37f68c6c21d

SIGQUIT: quit
PC=0x6e2b0 m=0 sigcode=0

goroutine 0 [idle]:
runtime.futex(0x2a5bcc, 0x80, 0x0, 0x0, 0x0, 0x0, 0x2a5850, 0x1, 0x19b38, 0x44cd0, ...)
	/workdir/go/src/runtime/sys_linux_arm.s:334 +0x1c
runtime.futexsleep(0x2a5bcc, 0x0, 0xffffffff, 0xffffffff)
	/workdir/go/src/runtime/os_linux.go:44 +0x70
runtime.notesleep(0x2a5bcc)
	/workdir/go/src/runtime/lock_futex.go:151 +0xac
runtime.stoplockedm()
	/workdir/go/src/runtime/proc.go:2068 +0x60
runtime.schedule()
	/workdir/go/src/runtime/proc.go:2469 +0x424
runtime.park_m(0x454ee0)
	/workdir/go/src/runtime/proc.go:2610 +0x80
runtime.mcall(0x6b6a4)
	/workdir/go/src/runtime/asm_arm.s:289 +0x5c

goroutine 1 [chan receive]:
testing.(*T).Run(0x4b7ae0, 0x190596, 0x11, 0x19abc8, 0x301)
	/workdir/go/src/testing/testing.go:961 +0x2cc
testing.runTests.func1(0x4b6000)
	/workdir/go/src/testing/testing.go:1202 +0x68
testing.tRunner(0x4b6000, 0x4286d0)
	/workdir/go/src/testing/testing.go:909 +0xa8
testing.runTests(0x48c070, 0x2a40b0, 0x7b, 0x7b, 0x0)
	/workdir/go/src/testing/testing.go:1200 +0x238
testing.(*M).Run(0x480080, 0x0)
	/workdir/go/src/testing/testing.go:1117 +0x13c
main.main()
	_testmain.go:318 +0x120

goroutine 33 [syscall]:
os/signal.signal_recv(0x0)
	/workdir/go/src/runtime/sigqueue.go:147 +0x130
os/signal.loop()
	/workdir/go/src/os/signal/signal_unix.go:23 +0x14
created by os/signal.init.0
	/workdir/go/src/os/signal/signal_unix.go:29 +0x30

goroutine 80 [runnable]:
os_test.TestProgWideChdir(0x4b7ae0)
	/workdir/go/src/os/os_test.go:1317 +0x368
testing.tRunner(0x4b7ae0, 0x19abc8)
	/workdir/go/src/testing/testing.go:909 +0xa8
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:960 +0x2ac

goroutine 81 [chan send]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x0)
	/workdir/go/src/os/os_test.go:1289 +0x11c
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 84 [chan send, locked to thread]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x3)
	/workdir/go/src/os/os_test.go:1289 +0x11c
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 86 [chan send, locked to thread]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x5)
	/workdir/go/src/os/os_test.go:1289 +0x11c
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 87 [chan send]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x6)
	/workdir/go/src/os/os_test.go:1289 +0x11c
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 88 [chan send, locked to thread]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x7)
	/workdir/go/src/os/os_test.go:1289 +0x11c
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 89 [runnable]:
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x8)
	/workdir/go/src/os/os_test.go:1272
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

goroutine 90 [runnable]:
runtime.LockOSThread()
	/workdir/go/src/runtime/proc.go:3537 +0x5c
os_test.TestProgWideChdir.func1(0x467940, 0x4b7ae0, 0x467980, 0x9)
	/workdir/go/src/os/os_test.go:1281 +0x124
created by os_test.TestProgWideChdir
	/workdir/go/src/os/os_test.go:1272 +0x84

trap    0x0
error   0x0
oldmask 0x0
r0      0x2a5bcc
r1      0x80
r2      0x0
r3      0x0
r4      0x0
r5      0x0
r6      0x0
r7      0xf0
r8      0x7
r9      0xe0
r10     0x2a5910
fp      0x7
ip      0x34
sp      0xbef4c86c
lr      0x3bad8
pc      0x6e2b0
cpsr    0x20000010
fault   0x0
*** Test killed with quit: ran too long (7m0s).
FAIL	os	420.014s

@bcmills bcmills changed the title os: occasional TestProgWideChdir deadlocks on openbsd/arm builder os: occasional TestProgWideChdir deadlocks on arm builders Jul 31, 2019
@rsc rsc modified the milestones: Go1.14, Backlog Oct 9, 2019
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. Testing An issue that has been verified to require only test changes, not just a test failure.
Projects
None yet
Milestone
Backlog
Development

No branches or pull requests
3 participants
@rsc @andybons @bcmills