@@ -753,11 +753,260 @@ func runtime_goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer
753753 return goroutineProfileWithLabels (p , labels )
754754}
755755
756+ const go119ConcurrentGoroutineProfile = true
757+
756758// labels may be nil. If labels is non-nil, it must have the same length as p.
757759func goroutineProfileWithLabels (p []StackRecord , labels []unsafe.Pointer ) (n int , ok bool ) {
758760 if labels != nil && len (labels ) != len (p ) {
759761 labels = nil
760762 }
763+
764+ if go119ConcurrentGoroutineProfile {
765+ return goroutineProfileWithLabelsConcurrent (p , labels )
766+ }
767+ return goroutineProfileWithLabelsSync (p , labels )
768+ }
769+
770+ var goroutineProfile = struct {
771+ sema uint32
772+ active bool
773+ offset atomic.Int64
774+ records []StackRecord
775+ labels []unsafe.Pointer
776+ }{
777+ sema : 1 ,
778+ }
779+
780+ // goroutineProfileState indicates the status of a goroutine's stack for the
781+ // current in-progress goroutine profile. Goroutines' stacks are initially
782+ // "Absent" from the profile, and end up "Satisfied" by the time the profile is
783+ // complete. While a goroutine's stack is being captured, its
784+ // goroutineProfileState will be "InProgress" and it will not be able to run
785+ // until the capture completes and the state moves to "Satisfied".
786+ //
787+ // Some goroutines (the finalizer goroutine, which at various times can be
788+ // either a "system" or a "user" goroutine, and the goroutine that is
789+ // coordinating the profile, any goroutines created during the profile) move
790+ // directly to the "Satisfied" state.
791+ type goroutineProfileState uint32
792+
793+ const (
794+ goroutineProfileAbsent goroutineProfileState = iota
795+ goroutineProfileInProgress
796+ goroutineProfileSatisfied
797+ )
798+
799+ type goroutineProfileStateHolder atomic.Uint32
800+
801+ func (p * goroutineProfileStateHolder ) Load () goroutineProfileState {
802+ return goroutineProfileState ((* atomic .Uint32 )(p ).Load ())
803+ }
804+
805+ func (p * goroutineProfileStateHolder ) Store (value goroutineProfileState ) {
806+ (* atomic .Uint32 )(p ).Store (uint32 (value ))
807+ }
808+
809+ func (p * goroutineProfileStateHolder ) CompareAndSwap (old , new goroutineProfileState ) bool {
810+ return (* atomic .Uint32 )(p ).CompareAndSwap (uint32 (old ), uint32 (new ))
811+ }
812+
813+ func goroutineProfileWithLabelsConcurrent (p []StackRecord , labels []unsafe.Pointer ) (n int , ok bool ) {
814+ semacquire (& goroutineProfile .sema )
815+
816+ ourg := getg ()
817+
818+ stopTheWorld ("profile" )
819+ // Using gcount while the world is stopped should give us a consistent view
820+ // of the number of live goroutines, minus the number of goroutines that are
821+ // alive and permanently marked as "system". But to make this count agree
822+ // with what we'd get from isSystemGoroutine, we need special handling for
823+ // goroutines that can vary between user and system to ensure that the count
824+ // doesn't change during the collection. So, check the finalizer goroutine
825+ // in particular.
826+ n = int (gcount ())
827+ if fingRunning {
828+ n ++
829+ }
830+
831+ if n > len (p ) {
832+ // There's not enough space in p to store the whole profile, so (per the
833+ // contract of runtime.GoroutineProfile) we're not allowed to write to p
834+ // at all and must return n, false.
835+ startTheWorld ()
836+ semrelease (& goroutineProfile .sema )
837+ return n , false
838+ }
839+
840+ // Save current goroutine.
841+ sp := getcallersp ()
842+ pc := getcallerpc ()
843+ systemstack (func () {
844+ saveg (pc , sp , ourg , & p [0 ])
845+ })
846+ ourg .goroutineProfiled .Store (goroutineProfileSatisfied )
847+ goroutineProfile .offset .Store (1 )
848+
849+ // Prepare for all other goroutines to enter the profile. Aside from ourg,
850+ // every goroutine struct in the allgs list has its goroutineProfiled field
851+ // cleared. Any goroutine created from this point on (while
852+ // goroutineProfile.active is set) will start with its goroutineProfiled
853+ // field set to goroutineProfileSatisfied.
854+ goroutineProfile .active = true
855+ goroutineProfile .records = p
856+ goroutineProfile .labels = labels
857+ // The finializer goroutine needs special handling because it can vary over
858+ // time between being a user goroutine (eligible for this profile) and a
859+ // system goroutine (to be excluded). Pick one before restarting the world.
860+ if fing != nil {
861+ fing .goroutineProfiled .Store (goroutineProfileSatisfied )
862+ }
863+ if readgstatus (fing ) != _Gdead && ! isSystemGoroutine (fing , false ) {
864+ doRecordGoroutineProfile (fing )
865+ }
866+ startTheWorld ()
867+
868+ // Visit each goroutine that existed as of the startTheWorld call above.
869+ //
870+ // New goroutines may not be in this list, but we didn't want to know about
871+ // them anyway. If they do appear in this list (via reusing a dead goroutine
872+ // struct, or racing to launch between the world restarting and us getting
873+ // the list), they will aleady have their goroutineProfiled field set to
874+ // goroutineProfileSatisfied before their state transitions out of _Gdead.
875+ //
876+ // Any goroutine that the scheduler tries to execute concurrently with this
877+ // call will start by adding itself to the profile (before the act of
878+ // executing can cause any changes in its stack).
879+ forEachGRace (func (gp1 * g ) {
880+ tryRecordGoroutineProfile (gp1 , Gosched )
881+ })
882+
883+ stopTheWorld ("profile cleanup" )
884+ endOffset := goroutineProfile .offset .Swap (0 )
885+ goroutineProfile .active = false
886+ goroutineProfile .records = nil
887+ goroutineProfile .labels = nil
888+ startTheWorld ()
889+
890+ // Restore the invariant that every goroutine struct in allgs has its
891+ // goroutineProfiled field cleared.
892+ forEachGRace (func (gp1 * g ) {
893+ gp1 .goroutineProfiled .Store (goroutineProfileAbsent )
894+ })
895+
896+ if raceenabled {
897+ raceacquire (unsafe .Pointer (& labelSync ))
898+ }
899+
900+ if n != int (endOffset ) {
901+ // It's a big surprise that the number of goroutines changed while we
902+ // were collecting the profile. But probably better to return a
903+ // truncated profile than to crash the whole process.
904+ //
905+ // For instance, needm moves a goroutine out of the _Gdead state and so
906+ // might be able to change the goroutine count without interacting with
907+ // the scheduler. For code like that, the race windows are small and the
908+ // combination of features is uncommon, so it's hard to be (and remain)
909+ // sure we've caught them all.
910+ }
911+
912+ semrelease (& goroutineProfile .sema )
913+ return n , true
914+ }
915+
916+ // tryRecordGoroutineProfileWB asserts that write barriers are allowed and calls
917+ // tryRecordGoroutineProfile.
918+ //
919+ //go:yeswritebarrierrec
920+ func tryRecordGoroutineProfileWB (gp1 * g ) {
921+ if getg ().m .p .ptr () == nil {
922+ throw ("no P available, write barriers are forbidden" )
923+ }
924+ tryRecordGoroutineProfile (gp1 , osyield )
925+ }
926+
927+ // tryRecordGoroutineProfile ensures that gp1 has the appropriate representation
928+ // in the current goroutine profile: either that it should not be profiled, or
929+ // that a snapshot of its call stack and labels are now in the profile.
930+ func tryRecordGoroutineProfile (gp1 * g , yield func ()) {
931+ if readgstatus (gp1 ) == _Gdead {
932+ // Dead goroutines should not appear in the profile. Goroutines that
933+ // start while profile collection is active will get goroutineProfiled
934+ // set to goroutineProfileSatisfied before transitioning out of _Gdead,
935+ // so here we check _Gdead first.
936+ return
937+ }
938+ if isSystemGoroutine (gp1 , true ) {
939+ // System goroutines should not appear in the profile. (The finalizer
940+ // goroutine is marked as "already profiled".)
941+ return
942+ }
943+
944+ for {
945+ prev := gp1 .goroutineProfiled .Load ()
946+ if prev == goroutineProfileSatisfied {
947+ // This goroutine is already in the profile (or is new since the
948+ // start of collection, so shouldn't appear in the profile).
949+ break
950+ }
951+ if prev == goroutineProfileInProgress {
952+ // Something else is adding gp1 to the goroutine profile right now.
953+ // Give that a moment to finish.
954+ yield ()
955+ continue
956+ }
957+
958+ // While we have gp1.goroutineProfiled set to
959+ // goroutineProfileInProgress, gp1 may appear _Grunnable but will not
960+ // actually be able to run. Disable preemption for ourselves, to make
961+ // sure we finish profiling gp1 right away instead of leaving it stuck
962+ // in this limbo.
963+ mp := acquirem ()
964+ if gp1 .goroutineProfiled .CompareAndSwap (goroutineProfileAbsent , goroutineProfileInProgress ) {
965+ doRecordGoroutineProfile (gp1 )
966+ gp1 .goroutineProfiled .Store (goroutineProfileSatisfied )
967+ }
968+ releasem (mp )
969+ }
970+ }
971+
972+ // doRecordGoroutineProfile writes gp1's call stack and labels to an in-progress
973+ // goroutine profile. Preemption is disabled.
974+ //
975+ // This may be called via tryRecordGoroutineProfile in two ways: by the
976+ // goroutine that is coordinating the goroutine profile (running on its own
977+ // stack), or from the scheduler in preparation to execute gp1 (running on the
978+ // system stack).
979+ func doRecordGoroutineProfile (gp1 * g ) {
980+ if readgstatus (gp1 ) == _Grunning {
981+ print ("doRecordGoroutineProfile gp1=" , gp1 .goid , "\n " )
982+ throw ("cannot read stack of running goroutine" )
983+ }
984+
985+ offset := int (goroutineProfile .offset .Add (1 )) - 1
986+
987+ if offset >= len (goroutineProfile .records ) {
988+ // Should be impossible, but better to return a truncated profile than
989+ // to crash the entire process at this point. Instead, deal with it in
990+ // goroutineProfileWithLabelsConcurrent where we have more context.
991+ return
992+ }
993+
994+ // saveg calls gentraceback, which may call cgo traceback functions. When
995+ // called from the scheduler, this is on the system stack already so
996+ // traceback.go:cgoContextPCs will avoid calling back into the scheduler.
997+ //
998+ // When called from the goroutine coordinating the profile, we still have
999+ // set gp1.goroutineProfiled to goroutineProfileInProgress and so are still
1000+ // preventing it from being truly _Grunnable. So we'll use the system stack
1001+ // to avoid schedule delays.
1002+ systemstack (func () { saveg (^ uintptr (0 ), ^ uintptr (0 ), gp1 , & goroutineProfile .records [offset ]) })
1003+
1004+ if goroutineProfile .labels != nil {
1005+ goroutineProfile .labels [offset ] = gp1 .labels
1006+ }
1007+ }
1008+
1009+ func goroutineProfileWithLabelsSync (p []StackRecord , labels []unsafe.Pointer ) (n int , ok bool ) {
7611010 gp := getg ()
7621011
7631012 isOK := func (gp1 * g ) bool {
0 commit comments