Skip to content

deep.processor.frame_collector

Processing for frame collection.

FrameCollector

This deals with collecting data from the paused frames.

Source code in deep/processor/frame_collector.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
class FrameCollector:
    """This deals with collecting data from the paused frames."""

    def __init__(self, source: FrameCollectorContext, frame: FrameType):
        """
        Create a new collector.

        :param source: the collector context
        :param frame:  the frame data
        """
        self.__has_time_exceeded = False
        self.__source = source
        self.__frame = frame

    def __time_exceeded(self) -> bool:
        if self.__has_time_exceeded:
            return self.__has_time_exceeded

        duration = (time_ns() - self.__source.ts) / 1000000  # make duration ms not ns
        self.__has_time_exceeded = duration > self.__source.max_tp_process_time
        return self.__has_time_exceeded

    def parse_short_name(self, filename) -> Tuple[str, bool]:
        """
        Process a file name into a shorter version.

        By default, the file names in python are the absolute path to the file on disk. These can be quite long,
        so we try to shorten the names by looking at the APP_ROOT and converting the file name into a relative path.

        e.g. if the file name is '/dev/python/custom_service/api/handler.py' and the APP_ROOT is
            '/dev/python/custom_service' then we shorten the path to 'custom_service/api/handler.py'.

        :param (str) filename: the file name
        :returns:
            (str) filename: the new file name
            (bool) is_app_frame: True if the file is an application frame file
        """
        is_app_frame, match = self.__source.is_app_frame(filename)
        if match is not None:
            return filename[len(match):], is_app_frame
        return filename, is_app_frame

    def collect(self, var_lookup: Dict[str, Variable], var_cache: VariableCacheProvider) \
            -> Tuple[List[StackFrame], Dict[str, Variable]]:
        """
        Collect the data from the current frame.

        :param var_lookup: the var lookup to use
        :param var_cache: the var cache to use
        :return:
        """
        current_frame = self.__frame
        collected_frames = []
        # while we still have frames process them
        while current_frame is not None:
            # process the current frame
            frame = self._process_frame(var_lookup, var_cache, current_frame,
                                        self.__source.should_collect_vars(len(collected_frames)))
            collected_frames.append(frame)
            current_frame = current_frame.f_back
        return collected_frames, var_lookup

    def _process_frame(self, var_lookup: Dict[str, Variable], var_cache: VariableCacheProvider,
                       frame: FrameType, collect_vars: bool) -> StackFrame:
        # process the current frame info
        lineno = frame.f_lineno
        filename = frame.f_code.co_filename
        func_name = frame.f_code.co_name

        f_locals = frame.f_locals
        _self = f_locals.get('self', None)
        class_name = None
        if _self is not None and hasattr(_self, '__class__'):
            class_name = _self.__class__.__name__

        var_ids = []
        # only process vars if we are under the time limit
        if collect_vars and not self.__time_exceeded():
            processor = VariableSetProcessor(var_lookup, var_cache, self.__source.collection_config)
            # we process the vars as a single dict of 'locals'
            variable, log_str = processor.process_variable("locals", f_locals)
            # now ee 'unwrap' the locals, so they are on the frame directly.
            if variable.vid in var_lookup:
                variable_val = var_lookup[variable.vid]
                del var_lookup[variable.vid]
                var_ids = variable_val.children
        short_path, app_frame = self.parse_short_name(filename)
        return StackFrame(filename, short_path, func_name, lineno, var_ids, class_name,
                          app_frame=app_frame)

__init__(source, frame)

Create a new collector.

:param source: the collector context :param frame: the frame data

Source code in deep/processor/frame_collector.py 
74
75
76
77
78
79
80
81
82
83
def __init__(self, source: FrameCollectorContext, frame: FrameType):
    """
    Create a new collector.

    :param source: the collector context
    :param frame:  the frame data
    """
    self.__has_time_exceeded = False
    self.__source = source
    self.__frame = frame

collect(var_lookup, var_cache)

Collect the data from the current frame.

:param var_lookup: the var lookup to use :param var_cache: the var cache to use :return:

Source code in deep/processor/frame_collector.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def collect(self, var_lookup: Dict[str, Variable], var_cache: VariableCacheProvider) \
        -> Tuple[List[StackFrame], Dict[str, Variable]]:
    """
    Collect the data from the current frame.

    :param var_lookup: the var lookup to use
    :param var_cache: the var cache to use
    :return:
    """
    current_frame = self.__frame
    collected_frames = []
    # while we still have frames process them
    while current_frame is not None:
        # process the current frame
        frame = self._process_frame(var_lookup, var_cache, current_frame,
                                    self.__source.should_collect_vars(len(collected_frames)))
        collected_frames.append(frame)
        current_frame = current_frame.f_back
    return collected_frames, var_lookup

parse_short_name(filename)

Process a file name into a shorter version.

By default, the file names in python are the absolute path to the file on disk. These can be quite long, so we try to shorten the names by looking at the APP_ROOT and converting the file name into a relative path.

e.g. if the file name is '/dev/python/custom_service/api/handler.py' and the APP_ROOT is '/dev/python/custom_service' then we shorten the path to 'custom_service/api/handler.py'.

:param (str) filename: the file name :returns: (str) filename: the new file name (bool) is_app_frame: True if the file is an application frame file

Source code in deep/processor/frame_collector.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def parse_short_name(self, filename) -> Tuple[str, bool]:
    """
    Process a file name into a shorter version.

    By default, the file names in python are the absolute path to the file on disk. These can be quite long,
    so we try to shorten the names by looking at the APP_ROOT and converting the file name into a relative path.

    e.g. if the file name is '/dev/python/custom_service/api/handler.py' and the APP_ROOT is
        '/dev/python/custom_service' then we shorten the path to 'custom_service/api/handler.py'.

    :param (str) filename: the file name
    :returns:
        (str) filename: the new file name
        (bool) is_app_frame: True if the file is an application frame file
    """
    is_app_frame, match = self.__source.is_app_frame(filename)
    if match is not None:
        return filename[len(match):], is_app_frame
    return filename, is_app_frame

FrameCollectorContext

Bases: ABC

The context that is used to wrap a collection event.

Source code in deep/processor/frame_collector.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
class FrameCollectorContext(abc.ABC):
    """The context that is used to wrap a collection event."""

    @property
    @abc.abstractmethod
    def max_tp_process_time(self) -> int:
        """The max time to spend processing a tracepoint."""
        pass

    @property
    @abc.abstractmethod
    def collection_config(self) -> VariableProcessorConfig:
        """The variable processing config."""
        pass

    @property
    @abc.abstractmethod
    def ts(self) -> int:
        """The timestamp in nanoseconds for this trigger."""
        pass

    @abc.abstractmethod
    def should_collect_vars(self, current_frame_index: int) -> bool:
        """
        Check if we can collect data for a frame.

        Frame indexes start from 0 (as the current frame) and increase as we go back up the stack.

        :param (int) current_frame_index: the current frame index.
        :return (bool): if we should collect the frame vars.
        """
        pass

    @abc.abstractmethod
    def is_app_frame(self, filename: str) -> Tuple[bool, str]:
        """
        Check if the current frame is a user application frame.

        :param filename: the frame file name
        :return: True if add frame, else False
        """
        pass

collection_config: VariableProcessorConfig abstractmethod property

The variable processing config.

max_tp_process_time: int abstractmethod property

The max time to spend processing a tracepoint.

ts: int abstractmethod property

The timestamp in nanoseconds for this trigger.

is_app_frame(filename) abstractmethod

Check if the current frame is a user application frame.

:param filename: the frame file name :return: True if add frame, else False

Source code in deep/processor/frame_collector.py 
60
61
62
63
64
65
66
67
68
@abc.abstractmethod
def is_app_frame(self, filename: str) -> Tuple[bool, str]:
    """
    Check if the current frame is a user application frame.

    :param filename: the frame file name
    :return: True if add frame, else False
    """
    pass

should_collect_vars(current_frame_index) abstractmethod

Check if we can collect data for a frame.

Frame indexes start from 0 (as the current frame) and increase as we go back up the stack.

:param (int) current_frame_index: the current frame index. :return (bool): if we should collect the frame vars.

Source code in deep/processor/frame_collector.py 
48
49
50
51
52
53
54
55
56
57
58
@abc.abstractmethod
def should_collect_vars(self, current_frame_index: int) -> bool:
    """
    Check if we can collect data for a frame.

    Frame indexes start from 0 (as the current frame) and increase as we go back up the stack.

    :param (int) current_frame_index: the current frame index.
    :return (bool): if we should collect the frame vars.
    """
    pass