Skip to content

sharing_kachery.py

kachery_download_file(uri, dest, kachery_zone_name)

Set the kachery resource url and attempt to download.

Source code in src/spyglass/sharing/sharing_kachery.py 
31
32
33
34
def kachery_download_file(uri: str, dest: str, kachery_zone_name: str) -> str:
    """Set the kachery resource url and attempt to download."""
    KacheryZone.set_resource_url({"kachery_zone_name": kachery_zone_name})
    return kcl.load_file(uri, dest=dest)

KacheryZone

Bases: SpyglassMixin, Manual

Source code in src/spyglass/sharing/sharing_kachery.py 
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
@schema
class KacheryZone(SpyglassMixin, dj.Manual):
    definition = """
    kachery_zone_name: varchar(200) # the name of the kachery zone. Note that this is the same as the name of the kachery resource.
    ---
    description: varchar(200) # description of this zone
    kachery_cloud_dir: varchar(200) # kachery cloud directory on local machine where files are linked
    kachery_proxy: varchar(200) # kachery sharing proxy
    -> Lab
    """

    @staticmethod
    def set_zone(key: dict):
        """Set the kachery zone based on the key to KacheryZone

        Parameters
        ----------
        key : dict
            key defining a single KacheryZone

        """
        try:
            kachery_zone_name, kachery_cloud_dir = (KacheryZone & key).fetch1(
                "kachery_zone_name", "kachery_cloud_dir"
            )
        except DataJointError:
            raise Exception(
                f"{key} does not correspond to a single entry in KacheryZone."
            )
            return None
        # set the new zone and cloud directory
        os.environ[kachery_zone_envar] = kachery_zone_name
        os.environ[kachery_cloud_dir_envar] = kachery_cloud_dir

    @staticmethod
    def reset_zone():
        """Resets the kachery zone environment variable to the default values."""
        if default_kachery_zone is not None:
            os.environ[kachery_zone_envar] = default_kachery_zone
        if default_kachery_cloud_dir is not None:
            os.environ[kachery_cloud_dir_envar] = default_kachery_cloud_dir

    @staticmethod
    def set_resource_url(key: dict):
        """Sets the KACHERY_RESOURCE_URL based on the key corresponding to a
        single Kachery Zone

        Parameters
        ----------
        key : dict
            key to retrieve a single kachery zone
        """
        try:
            kachery_zone_name, kachery_proxy = (KacheryZone & key).fetch1(
                "kachery_zone_name", "kachery_proxy"
            )
        except DataJointError:
            raise Exception(
                f"{key} does not correspond to a single entry in KacheryZone."
            )
        # set the new zone and cloud directory
        os.environ[kachery_zone_envar] = kachery_zone_name
        os.environ[kachery_resource_url_envar] = (
            kachery_proxy + "/r/" + kachery_zone_name
        )

    @staticmethod
    def reset_resource_url():
        """Resets the KACHERY_RESOURCE_URL to the default value."""
        KacheryZone.reset_zone()
        if default_kachery_resource_url is not None:
            os.environ[kachery_resource_url_envar] = (
                default_kachery_resource_url
            )

set_zone(key) staticmethod

Set the kachery zone based on the key to KacheryZone

Parameters:

Name Type Description Default
key dict

key defining a single KacheryZone

 required
Source code in src/spyglass/sharing/sharing_kachery.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
@staticmethod
def set_zone(key: dict):
    """Set the kachery zone based on the key to KacheryZone

    Parameters
    ----------
    key : dict
        key defining a single KacheryZone

    """
    try:
        kachery_zone_name, kachery_cloud_dir = (KacheryZone & key).fetch1(
            "kachery_zone_name", "kachery_cloud_dir"
        )
    except DataJointError:
        raise Exception(
            f"{key} does not correspond to a single entry in KacheryZone."
        )
        return None
    # set the new zone and cloud directory
    os.environ[kachery_zone_envar] = kachery_zone_name
    os.environ[kachery_cloud_dir_envar] = kachery_cloud_dir

reset_zone() staticmethod

Resets the kachery zone environment variable to the default values.

Source code in src/spyglass/sharing/sharing_kachery.py 
71
72
73
74
75
76
77
@staticmethod
def reset_zone():
    """Resets the kachery zone environment variable to the default values."""
    if default_kachery_zone is not None:
        os.environ[kachery_zone_envar] = default_kachery_zone
    if default_kachery_cloud_dir is not None:
        os.environ[kachery_cloud_dir_envar] = default_kachery_cloud_dir

set_resource_url(key) staticmethod

Sets the KACHERY_RESOURCE_URL based on the key corresponding to a single Kachery Zone

Parameters:

Name Type Description Default
key dict

key to retrieve a single kachery zone

 required
Source code in src/spyglass/sharing/sharing_kachery.py 
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
@staticmethod
def set_resource_url(key: dict):
    """Sets the KACHERY_RESOURCE_URL based on the key corresponding to a
    single Kachery Zone

    Parameters
    ----------
    key : dict
        key to retrieve a single kachery zone
    """
    try:
        kachery_zone_name, kachery_proxy = (KacheryZone & key).fetch1(
            "kachery_zone_name", "kachery_proxy"
        )
    except DataJointError:
        raise Exception(
            f"{key} does not correspond to a single entry in KacheryZone."
        )
    # set the new zone and cloud directory
    os.environ[kachery_zone_envar] = kachery_zone_name
    os.environ[kachery_resource_url_envar] = (
        kachery_proxy + "/r/" + kachery_zone_name
    )

reset_resource_url() staticmethod

Resets the KACHERY_RESOURCE_URL to the default value.

Source code in src/spyglass/sharing/sharing_kachery.py 
103
104
105
106
107
108
109
110
@staticmethod
def reset_resource_url():
    """Resets the KACHERY_RESOURCE_URL to the default value."""
    KacheryZone.reset_zone()
    if default_kachery_resource_url is not None:
        os.environ[kachery_resource_url_envar] = (
            default_kachery_resource_url
        )

AnalysisNwbfileKachery

Bases: SpyglassMixin, Computed

Source code in src/spyglass/sharing/sharing_kachery.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
@schema
class AnalysisNwbfileKachery(SpyglassMixin, dj.Computed):
    definition = """
    -> AnalysisNwbfileKacherySelection
    ---
    analysis_file_uri='': varchar(200)  # the uri of the file
    """

    class LinkedFile(SpyglassMixin, dj.Part):
        definition = """
        -> AnalysisNwbfileKachery
        linked_file_rel_path: varchar(200) # the path for the linked file relative to the SPYGLASS_BASE_DIR environment variable
        ---
        linked_file_uri='': varchar(200) # the uri for the linked file
        """

    def make(self, key):
        """Populate with the uri of the analysis file"""
        # note that we're assuming that the user has initialized a kachery-cloud
        # client with kachery-cloud-init. Uncomment the line below once we are
        # sharing linked files as well.

        # linked_key = copy.deepcopy(key)

        logger.info(f'Linking {key["analysis_file_name"]} in kachery-cloud...')
        # set the kachery zone

        KacheryZone.set_zone(key)

        key["analysis_file_uri"] = kcl.link_file(
            AnalysisNwbfile().get_abs_path(key["analysis_file_name"])
        )
        logger.info(
            os.environ[kachery_zone_envar], os.environ[kachery_cloud_dir_envar]
        )
        logger.info(AnalysisNwbfile().get_abs_path(key["analysis_file_name"]))
        logger.info(kcl.load_file(key["analysis_file_uri"]))
        self.insert1(key)

        # we also need to insert any linked files
        # TODO: change this to automatically detect all linked files
        # self.LinkedFile.insert1(key)

        # reset the Kachery zone and cloud_dir to the defaults
        KacheryZone.reset_zone()

    @staticmethod
    def download_file(
        analysis_file_name: str, permit_fail: bool = False
    ) -> bool:
        """Download the specified analysis file and associated linked files
        from kachery-cloud if possible

        Parameters
        ----------
        analysis_file_name : str
            The name of the analysis file

        Returns
        ----------
        is_success : bool
            True if the file was successfully downloaded, False otherwise
        """
        fetched_list = (
            AnalysisNwbfileKachery & {"analysis_file_name": analysis_file_name}
        ).fetch("analysis_file_uri", "kachery_zone_name")
        downloaded = False
        for uri, kachery_zone_name in zip(fetched_list[0], fetched_list[1]):
            if len(uri) == 0:
                return False
            logger.info("uri:", uri)
            if kachery_download_file(
                uri=uri,
                dest=AnalysisNwbfile.get_abs_path(analysis_file_name),
                kachery_zone_name=kachery_zone_name,
            ):
                downloaded = True
                # now download the linked file(s)
                linked_files = (
                    AnalysisNwbfileKachery.LinkedFile
                    & {"analysis_file_name": analysis_file_name}
                ).fetch(as_dict=True)
                for file in linked_files:
                    uri = file["linked_file_uri"]
                    logger.info(f"attempting to download linked file uri {uri}")
                    linked_file_path = (
                        os.environ["SPYGLASS_BASE_DIR"]
                        + file["linked_file_rel_path"]
                    )
                    if not kachery_download_file(
                        uri=uri,
                        dest=linked_file_path,
                        kachery_zone_name=kachery_zone_name,
                    ):
                        raise Exception(
                            f"Linked file {linked_file_path} cannot be downloaded"
                        )
        if not downloaded and not permit_fail:
            raise Exception(f"{analysis_file_name} cannot be downloaded")

        return downloaded

make(key)

Populate with the uri of the analysis file

Source code in src/spyglass/sharing/sharing_kachery.py 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
def make(self, key):
    """Populate with the uri of the analysis file"""
    # note that we're assuming that the user has initialized a kachery-cloud
    # client with kachery-cloud-init. Uncomment the line below once we are
    # sharing linked files as well.

    # linked_key = copy.deepcopy(key)

    logger.info(f'Linking {key["analysis_file_name"]} in kachery-cloud...')
    # set the kachery zone

    KacheryZone.set_zone(key)

    key["analysis_file_uri"] = kcl.link_file(
        AnalysisNwbfile().get_abs_path(key["analysis_file_name"])
    )
    logger.info(
        os.environ[kachery_zone_envar], os.environ[kachery_cloud_dir_envar]
    )
    logger.info(AnalysisNwbfile().get_abs_path(key["analysis_file_name"]))
    logger.info(kcl.load_file(key["analysis_file_uri"]))
    self.insert1(key)

    # we also need to insert any linked files
    # TODO: change this to automatically detect all linked files
    # self.LinkedFile.insert1(key)

    # reset the Kachery zone and cloud_dir to the defaults
    KacheryZone.reset_zone()

download_file(analysis_file_name, permit_fail=False) staticmethod

Download the specified analysis file and associated linked files from kachery-cloud if possible

Parameters:

Name Type Description Default
analysis_file_name str

The name of the analysis file

 required

Returns:

Name Type Description
is_success bool

True if the file was successfully downloaded, False otherwise
Source code in src/spyglass/sharing/sharing_kachery.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
@staticmethod
def download_file(
    analysis_file_name: str, permit_fail: bool = False
) -> bool:
    """Download the specified analysis file and associated linked files
    from kachery-cloud if possible

    Parameters
    ----------
    analysis_file_name : str
        The name of the analysis file

    Returns
    ----------
    is_success : bool
        True if the file was successfully downloaded, False otherwise
    """
    fetched_list = (
        AnalysisNwbfileKachery & {"analysis_file_name": analysis_file_name}
    ).fetch("analysis_file_uri", "kachery_zone_name")
    downloaded = False
    for uri, kachery_zone_name in zip(fetched_list[0], fetched_list[1]):
        if len(uri) == 0:
            return False
        logger.info("uri:", uri)
        if kachery_download_file(
            uri=uri,
            dest=AnalysisNwbfile.get_abs_path(analysis_file_name),
            kachery_zone_name=kachery_zone_name,
        ):
            downloaded = True
            # now download the linked file(s)
            linked_files = (
                AnalysisNwbfileKachery.LinkedFile
                & {"analysis_file_name": analysis_file_name}
            ).fetch(as_dict=True)
            for file in linked_files:
                uri = file["linked_file_uri"]
                logger.info(f"attempting to download linked file uri {uri}")
                linked_file_path = (
                    os.environ["SPYGLASS_BASE_DIR"]
                    + file["linked_file_rel_path"]
                )
                if not kachery_download_file(
                    uri=uri,
                    dest=linked_file_path,
                    kachery_zone_name=kachery_zone_name,
                ):
                    raise Exception(
                        f"Linked file {linked_file_path} cannot be downloaded"
                    )
    if not downloaded and not permit_fail:
        raise Exception(f"{analysis_file_name} cannot be downloaded")

    return downloaded

share_data_to_kachery(restriction={}, table_list=[], zone_name=None)

Share data to kachery

Parameters:

Name Type Description Default
restriction dict

restriction to select what data should be shared from table, by default {}

 {}
table_list list

List of tables to share data from, by default []

 []
zone_name str

What kachery zone to share the data to, by default zone in spyglass.settings.config, which looks for KACHERY_ZONE environmental variable, but defaults to 'franklab.default'

 None

Raises:

Type Description
ValueError

Does not allow sharing of all data in table
Source code in src/spyglass/sharing/sharing_kachery.py 
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def share_data_to_kachery(
    restriction={},
    table_list=[],
    zone_name=None,
):
    """Share data to kachery

    Parameters
    ----------
    restriction : dict, optional
        restriction to select what data should be shared from table, by default {}
    table_list : list, optional
        List of tables to share data from, by default []
    zone_name : str, optional
        What kachery zone to share the data to, by default zone in spyglass.settings.config,
        which looks for `KACHERY_ZONE` environmental variable, but defaults to
        'franklab.default'

    Raises
    ------
    ValueError
        Does not allow sharing of all data in table
    """
    if not zone_name:
        zone_name = config["KACHERY_ZONE"]
    kachery_selection_key = {"kachery_zone_name": zone_name}
    if not restriction:
        raise ValueError("Must provide a restriction to the table")
    selection_inserts = []
    for table in table_list:
        analysis_file_list = (table & restriction).fetch("analysis_file_name")
        for file in analysis_file_list:  # Add all analysis to shared list
            kachery_selection_key["analysis_file_name"] = file
            selection_inserts.append(kachery_selection_key)
    AnalysisNwbfileKacherySelection.insert(
        selection_inserts, skip_duplicates=True
    )
    AnalysisNwbfileKachery.populate()