Skip to content

lfp_artifact.py

LFPArtifactDetectionParameters

Bases: SpyglassMixin, Manual

Parameters for detecting LFP artifacts.

Parameters:

Name Type Description Default
artifact_params_name str

Name of the artifact detection parameters.

 required
artifact_params dict

artifact_detection_algorithm: str artifact_detection_algorithm_params: dict amplitude_thresh_1st : float, optional Amplitude (ad units) threshold for exclusion, should be >=0, defaults to None amplitude_thresh_2nd : float, optional Amplitude (ad units) threshold for exclusion, should be >=0, defaults to None proportion_above_thresh_1st : float, optional, should be>0 and <=1. Proportion of electrodes that need to have threshold crossings, defaults to 1 proportion_above_thresh_2nd : float, optional, should be>0 and <=1 Proportion of electrodes that need to have threshold crossings, defaults to 1 removal_window_ms : float, optional Width of the window in milliseconds to mask out per artifact (window/2 removed on each side of threshold crossing), defaults to 1 ms local_window_ms: float, optional Width of the local time window in milliseconds used for artifact detection. This parameter defines the duration of the local analysis window, defaults to None. referencing: dict, optional ref_on: bool reference_list: list of int Reference electrode IDs. electrode_list: list of int Electrode IDs to be referenced.

 required
Source code in src/spyglass/lfp/v1/lfp_artifact.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
@schema
class LFPArtifactDetectionParameters(SpyglassMixin, dj.Manual):
    """Parameters for detecting LFP artifacts.

    Parameters
    ----------
    artifact_params_name : str
        Name of the artifact detection parameters.
    artifact_params : dict
        artifact_detection_algorithm: str
        artifact_detection_algorithm_params: dict
            amplitude_thresh_1st : float, optional
                Amplitude (ad units) threshold for exclusion, should be >=0,
                defaults to None
            amplitude_thresh_2nd : float, optional
                Amplitude (ad units) threshold for exclusion, should be >=0,
                defaults to None
            proportion_above_thresh_1st : float, optional,
                should be>0 and <=1. Proportion of electrodes that need to have
                threshold crossings, defaults to 1
            proportion_above_thresh_2nd : float, optional, should be>0 and <=1
                Proportion of electrodes that need to have threshold crossings,
                defaults to 1
            removal_window_ms : float, optional
                Width of the window in milliseconds to mask out per artifact
                (window/2 removed on each side of threshold crossing), defaults
                to 1 ms
            local_window_ms: float, optional
                Width of the local time window in milliseconds used for artifact
                detection. This parameter defines the duration of the local
                analysis window, defaults to None.
        referencing: dict, optional
            ref_on: bool
            reference_list: list of int
                Reference electrode IDs.
            electrode_list: list of int
                Electrode IDs to be referenced.
    """

    definition = """
    # Parameters for detecting LFP artifact times within a LFP group.
    artifact_params_name: varchar(64)
    ---
    artifact_params: blob  # dictionary of parameters
    """

    # See #630, #664. Excessive key length.

    def insert_default(self):
        """Insert the default artifact parameters."""
        diff_params = [
            "default_difference",
            {
                "artifact_detection_algorithm": "difference",
                "artifact_detection_algorithm_params": {
                    "amplitude_thresh_1st": 500,  # must be None or >= 0
                    "proportion_above_thresh_1st": 0.1,
                    "amplitude_thresh_2nd": 1000,  # must be None or >= 0
                    "proportion_above_thresh_2nd": 0.05,
                    "removal_window_ms": 10.0,  # in milliseconds
                    "local_window_ms": 40.0,  # in milliseconds
                },
            },
        ]

        diff_ref_params = [
            "default_difference_ref",
            {
                "artifact_detection_algorithm": "difference",
                "artifact_detection_algorithm_params": {
                    "amplitude_thresh_1st": 500,  # must be None or >= 0
                    "proportion_above_thresh_1st": 0.1,
                    "amplitude_thresh_2nd": 1000,  # must be None or >= 0
                    "proportion_above_thresh_2nd": 0.05,
                    "removal_window_ms": 10.0,  # in milliseconds
                    "local_window_ms": 40.0,  # in milliseconds
                },
                "referencing": {
                    "ref_on": 1,
                    "reference_list": [0, 0, 0, 0, 0],
                    "electrode_list": [0, 0],
                },
            },
        ]

        no_params = [
            "none",
            {
                "artifact_detection_algorithm": "difference",
                "artifact_detection_algorithm_params": {
                    "amplitude_thresh_1st": None,  # must be None or >= 0
                    "proportion_above_thresh_1st": None,
                    "amplitude_thresh_2nd": None,  # must be None or >= 0
                    "proportion_above_thresh_2nd": None,
                    "removal_window_ms": None,  # in milliseconds
                    "local_window_ms": None,  # in milliseconds
                },
            },
        ]

        mad_params = [
            "default_mad",
            {
                "artifact_detection_algorithm": "mad",
                "artifact_detection_algorithm_params": {
                    # akin to z-score std dev if the distribution is normal
                    "mad_thresh": 6.0,
                    "proportion_above_thresh": 0.1,
                    "removal_window_ms": 10.0,  # in milliseconds
                },
            },
        ]

        self.insert(
            [diff_params, diff_ref_params, no_params, mad_params],
            skip_duplicates=True,
        )

insert_default()

Insert the default artifact parameters.

Source code in src/spyglass/lfp/v1/lfp_artifact.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def insert_default(self):
    """Insert the default artifact parameters."""
    diff_params = [
        "default_difference",
        {
            "artifact_detection_algorithm": "difference",
            "artifact_detection_algorithm_params": {
                "amplitude_thresh_1st": 500,  # must be None or >= 0
                "proportion_above_thresh_1st": 0.1,
                "amplitude_thresh_2nd": 1000,  # must be None or >= 0
                "proportion_above_thresh_2nd": 0.05,
                "removal_window_ms": 10.0,  # in milliseconds
                "local_window_ms": 40.0,  # in milliseconds
            },
        },
    ]

    diff_ref_params = [
        "default_difference_ref",
        {
            "artifact_detection_algorithm": "difference",
            "artifact_detection_algorithm_params": {
                "amplitude_thresh_1st": 500,  # must be None or >= 0
                "proportion_above_thresh_1st": 0.1,
                "amplitude_thresh_2nd": 1000,  # must be None or >= 0
                "proportion_above_thresh_2nd": 0.05,
                "removal_window_ms": 10.0,  # in milliseconds
                "local_window_ms": 40.0,  # in milliseconds
            },
            "referencing": {
                "ref_on": 1,
                "reference_list": [0, 0, 0, 0, 0],
                "electrode_list": [0, 0],
            },
        },
    ]

    no_params = [
        "none",
        {
            "artifact_detection_algorithm": "difference",
            "artifact_detection_algorithm_params": {
                "amplitude_thresh_1st": None,  # must be None or >= 0
                "proportion_above_thresh_1st": None,
                "amplitude_thresh_2nd": None,  # must be None or >= 0
                "proportion_above_thresh_2nd": None,
                "removal_window_ms": None,  # in milliseconds
                "local_window_ms": None,  # in milliseconds
            },
        },
    ]

    mad_params = [
        "default_mad",
        {
            "artifact_detection_algorithm": "mad",
            "artifact_detection_algorithm_params": {
                # akin to z-score std dev if the distribution is normal
                "mad_thresh": 6.0,
                "proportion_above_thresh": 0.1,
                "removal_window_ms": 10.0,  # in milliseconds
            },
        },
    ]

    self.insert(
        [diff_params, diff_ref_params, no_params, mad_params],
        skip_duplicates=True,
    )

LFPArtifactDetection

Bases: SpyglassMixin, Computed

Source code in src/spyglass/lfp/v1/lfp_artifact.py 
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
@schema
class LFPArtifactDetection(SpyglassMixin, dj.Computed):
    definition = """
    # Stores artifact times and valid no-artifact times as intervals.
    -> LFPArtifactDetectionSelection
    ---
    artifact_times: longblob # np array of artifact intervals
    artifact_removed_valid_times: longblob # np array of no-artifact intervals
    artifact_removed_interval_list_name: varchar(200)
        # name of the array of no-artifact valid time intervals
    """

    def make(self, key):
        """Populate the LFPArtifactDetection table with artifact times.

        1. Fetch parameters and LFP data from LFPArtifactDetectionParameters
            and LFPV1, respectively.
        2. Optionally reference the LFP data.
        3. Pass data to chosen artifact detection algorithm.
        3. Insert into LFPArtifactRemovedIntervalList, IntervalList, and
            LFPArtifactDetection.
        """
        artifact_params = (
            LFPArtifactDetectionParameters
            & {"artifact_params_name": key["artifact_params_name"]}
        ).fetch1("artifact_params")

        algorithm = artifact_params["artifact_detection_algorithm"]
        params = artifact_params["artifact_detection_algorithm_params"]

        # get LFP data
        lfp_eseries = (LFPV1 & key).fetch_nwb()[0]["lfp"]
        sampling_frequency = (LFPV1 & key).fetch("lfp_sampling_rate")[0]
        lfp_data = np.asarray(
            lfp_eseries.data[:, :],
            dtype=type(lfp_eseries.data[0][0]),
        )

        is_diff = algorithm == "difference"
        # do referencing at this step
        if "referencing" in artifact_params:
            ref = artifact_params["referencing"]["ref_on"] if is_diff else None
            lfp_band_ref_id = artifact_params["referencing"]["reference_list"]
            if artifact_params["referencing"]["ref_on"]:
                lfp_band_ref_index = get_electrode_indices(
                    lfp_eseries, lfp_band_ref_id
                )
                lfp_band_elect_index = get_electrode_indices(
                    lfp_eseries,
                    artifact_params["referencing"]["electrode_list"],
                )

                # maybe this lfp_elec_list is supposed to be a list on indices
                for index, elect_index in enumerate(lfp_band_elect_index):
                    if lfp_band_ref_id[index] == -1:
                        continue
                    lfp_data[:, elect_index] = (
                        lfp_data[:, elect_index]
                        - lfp_data[:, lfp_band_ref_index[index]]
                    )
        else:
            ref = False if is_diff else None

        data = lfp_data if is_diff else lfp_eseries

        (
            artifact_removed_valid_times,
            artifact_times,
        ) = ARTIFACT_DETECTION_ALGORITHMS[algorithm](
            data,
            **params,
            sampling_frequency=sampling_frequency,
            timestamps=lfp_eseries.timestamps if is_diff else None,
            referencing=ref,
        )

        key.update(
            dict(
                artifact_times=artifact_times,
                artifact_removed_valid_times=artifact_removed_valid_times,
                artifact_removed_interval_list_name=uuid.uuid4(),
            )
        )

        interval_key = {  # also insert into IntervalList
            "nwb_file_name": key["nwb_file_name"],
            "interval_list_name": key["artifact_removed_interval_list_name"],
            "valid_times": key["artifact_removed_valid_times"],
            "pipeline": self.full_table_name,
        }

        LFPArtifactRemovedIntervalList.insert1(key)
        IntervalList.insert1(interval_key)
        self.insert1(key)

make(key)

Populate the LFPArtifactDetection table with artifact times.

  1. Fetch parameters and LFP data from LFPArtifactDetectionParameters and LFPV1, respectively.
  2. Optionally reference the LFP data.
  3. Pass data to chosen artifact detection algorithm.
  4. Insert into LFPArtifactRemovedIntervalList, IntervalList, and LFPArtifactDetection.
Source code in src/spyglass/lfp/v1/lfp_artifact.py 
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def make(self, key):
    """Populate the LFPArtifactDetection table with artifact times.

    1. Fetch parameters and LFP data from LFPArtifactDetectionParameters
        and LFPV1, respectively.
    2. Optionally reference the LFP data.
    3. Pass data to chosen artifact detection algorithm.
    3. Insert into LFPArtifactRemovedIntervalList, IntervalList, and
        LFPArtifactDetection.
    """
    artifact_params = (
        LFPArtifactDetectionParameters
        & {"artifact_params_name": key["artifact_params_name"]}
    ).fetch1("artifact_params")

    algorithm = artifact_params["artifact_detection_algorithm"]
    params = artifact_params["artifact_detection_algorithm_params"]

    # get LFP data
    lfp_eseries = (LFPV1 & key).fetch_nwb()[0]["lfp"]
    sampling_frequency = (LFPV1 & key).fetch("lfp_sampling_rate")[0]
    lfp_data = np.asarray(
        lfp_eseries.data[:, :],
        dtype=type(lfp_eseries.data[0][0]),
    )

    is_diff = algorithm == "difference"
    # do referencing at this step
    if "referencing" in artifact_params:
        ref = artifact_params["referencing"]["ref_on"] if is_diff else None
        lfp_band_ref_id = artifact_params["referencing"]["reference_list"]
        if artifact_params["referencing"]["ref_on"]:
            lfp_band_ref_index = get_electrode_indices(
                lfp_eseries, lfp_band_ref_id
            )
            lfp_band_elect_index = get_electrode_indices(
                lfp_eseries,
                artifact_params["referencing"]["electrode_list"],
            )

            # maybe this lfp_elec_list is supposed to be a list on indices
            for index, elect_index in enumerate(lfp_band_elect_index):
                if lfp_band_ref_id[index] == -1:
                    continue
                lfp_data[:, elect_index] = (
                    lfp_data[:, elect_index]
                    - lfp_data[:, lfp_band_ref_index[index]]
                )
    else:
        ref = False if is_diff else None

    data = lfp_data if is_diff else lfp_eseries

    (
        artifact_removed_valid_times,
        artifact_times,
    ) = ARTIFACT_DETECTION_ALGORITHMS[algorithm](
        data,
        **params,
        sampling_frequency=sampling_frequency,
        timestamps=lfp_eseries.timestamps if is_diff else None,
        referencing=ref,
    )

    key.update(
        dict(
            artifact_times=artifact_times,
            artifact_removed_valid_times=artifact_removed_valid_times,
            artifact_removed_interval_list_name=uuid.uuid4(),
        )
    )

    interval_key = {  # also insert into IntervalList
        "nwb_file_name": key["nwb_file_name"],
        "interval_list_name": key["artifact_removed_interval_list_name"],
        "valid_times": key["artifact_removed_valid_times"],
        "pipeline": self.full_table_name,
    }

    LFPArtifactRemovedIntervalList.insert1(key)
    IntervalList.insert1(interval_key)
    self.insert1(key)