[PATCH 07 of 10 V2] statprof: support stacked collection

[Yuya Nishihara]

On Wed, 17 Aug 2016 09:03:45 -0700, Gregory Szorc wrote:
> # HG changeset patch
> # User Gregory Szorc <gregory.szorc at gmail.com>
> # Date 1471449673 25200
> #      Wed Aug 17 09:01:13 2016 -0700
> # Node ID bf247c2f0136d23100b0098fe152cfabbb511c1b
> # Parent  7409c675a2be190605f07277627b095976f56da8
> statprof: support stacked collection

> -def samplerthread(tid):
> +def samplerthread():
>      while not stopthread.is_set():
>          state.accumulate_time(clock())
>  
> -        frame = sys._current_frames()[tid]
> +        frame = sys._current_frames()[state.threadid]
>          state.samples.append(Sample.from_frame(frame, state.accumulated_time))

[snip]

>  def start(mechanism='thread'):
>      '''Install the profiling signal handler, and start profiling.'''
> +    # Store old state if present.
> +    if state.profile_level > 0:
> +        laststate = {
> +            'samples': state.samples,
> +            'tid': state.threadid,
> +            'accumulated_time': state.accumulated_time,
> +        }
> +        _statestack.append(laststate)
> +
> +    state.samples = []
> +    state.accumulated_time = 0.0
> +    frame = inspect.currentframe()
> +    tid = [k for k, f in sys._current_frames().items() if f == frame][0]
> +    state.threadid = tid
> +
>      state.profile_level += 1
>      if state.profile_level == 1:
>          state.last_start_time = clock()
>          rpt = state.remaining_prof_time
>          state.remaining_prof_time = None
>  
>          global lastmechanism
>          lastmechanism = mechanism
>  
>          if mechanism == 'signal':
>              signal.signal(signal.SIGPROF, profile_signal_handler)
>              signal.setitimer(signal.ITIMER_PROF,
>                  rpt or state.sample_interval, 0.0)
>          elif mechanism == 'thread':
> -            frame = inspect.currentframe()
> -            tid = [k for k, f in sys._current_frames().items() if f == frame][0]
> +
>              state.thread = threading.Thread(target=samplerthread,
> -                                 args=(tid,), name="samplerthread")
> +                                            name="samplerthread")
>              state.thread.start()
>  
>  def stop():
>      '''Stop profiling, and uninstall the profiling signal handler.'''
>      state.profile_level -= 1
>      if state.profile_level == 0:
>          if lastmechanism == 'signal':
>              rpt = signal.setitimer(signal.ITIMER_PROF, 0.0, 0.0)
> @@ -317,16 +338,39 @@ def stop():
>              state.thread.join()
>  
>          state.accumulate_time(clock())
>          state.last_start_time = None
>          statprofpath = os.environ.get('STATPROF_DEST')
>          if statprofpath:
>              save_data(statprofpath)
>  
> +        return CollectedData(state.samples, state.accumulated_time,
> +                             state.sample_interval)
> +    elif state.profile_level > 0:
> +        newstate = _statestack.pop()
> +
> +        data = CollectedData(state.samples, state.accumulated_time,
> +                             state.sample_interval)
> +
> +        # Merge the just collected data onto the base state since the base
> +        # state was active while this one was collecting.
> +        newstate['samples'].extend(state.samples)
> +        newstate['threadid'] = newstate['tid']
> +        newstate['accumulated_time'] += state.accumulated_time
> +
> +        # Now make the base state the active state.
> +        state.samples = newstate['samples']
> +        state.threadid = newstate['tid']
> +        state.accumulated_time = newstate['accumulated_time']
> +
> +        return data

I don't think states can form a stack because start/stop will be called by
arbitrary threads. Instead, samplerthread would need to record activities of
all threads of profiling enabled.

But, in either ways, I don't think statprof can show a good result if
more than one threads are actively running and switched at random.

The other patches look generally good to me.