Skip to content

sharing_kachery.py

kachery_download_file(uri, dest, kachery_zone_name)

Set the kachery resource url and attempt to download.

Source code in src/spyglass/sharing/sharing_kachery.py
def kachery_download_file(uri: str, dest: str, kachery_zone_name: str) -> str:
    """Set the kachery resource url and attempt to download."""
    KacheryZone.set_resource_url({"kachery_zone_name": kachery_zone_name})
    return kcl.load_file(uri, dest=dest)

KacheryZone

Bases: SpyglassMixin, Manual

Source code in src/spyglass/sharing/sharing_kachery.py
@schema
class KacheryZone(SpyglassMixin, dj.Manual):
    definition = """
    kachery_zone_name: varchar(200) # the name of the kachery zone. Note that this is the same as the name of the kachery resource.
    ---
    description: varchar(200) # description of this zone
    kachery_cloud_dir: varchar(200) # kachery cloud directory on local machine where files are linked
    kachery_proxy: varchar(200) # kachery sharing proxy
    -> Lab
    """

    @staticmethod
    def set_zone(key: dict):
        """Set the kachery zone based on the key to KacheryZone

        Parameters
        ----------
        key : dict
            key defining a single KacheryZone

        """
        try:
            kachery_zone_name, kachery_cloud_dir = (KacheryZone & key).fetch1(
                "kachery_zone_name", "kachery_cloud_dir"
            )
        except DataJointError:
            raise Exception(
                f"{key} does not correspond to a single entry in KacheryZone."
            )
            return None
        # set the new zone and cloud directory
        os.environ[kachery_zone_envar] = kachery_zone_name
        os.environ[kachery_cloud_dir_envar] = kachery_cloud_dir

    @staticmethod
    def reset_zone():
        """Resets the kachery zone environment variable to the default values."""
        if default_kachery_zone is not None:
            os.environ[kachery_zone_envar] = default_kachery_zone
        if default_kachery_cloud_dir is not None:
            os.environ[kachery_cloud_dir_envar] = default_kachery_cloud_dir

    @staticmethod
    def set_resource_url(key: dict):
        """Sets the KACHERY_RESOURCE_URL based on the key corresponding to a
        single Kachery Zone

        Parameters
        ----------
        key : dict
            key to retrieve a single kachery zone
        """
        try:
            kachery_zone_name, kachery_proxy = (KacheryZone & key).fetch1(
                "kachery_zone_name", "kachery_proxy"
            )
        except DataJointError:
            raise Exception(
                f"{key} does not correspond to a single entry in KacheryZone."
            )
        # set the new zone and cloud directory
        os.environ[kachery_zone_envar] = kachery_zone_name
        os.environ[kachery_resource_url_envar] = (
            kachery_proxy + "/r/" + kachery_zone_name
        )

    @staticmethod
    def reset_resource_url():
        """Resets the KACHERY_RESOURCE_URL to the default value."""
        KacheryZone.reset_zone()
        if default_kachery_resource_url is not None:
            os.environ[kachery_resource_url_envar] = (
                default_kachery_resource_url
            )

set_zone(key) staticmethod

Set the kachery zone based on the key to KacheryZone

Parameters:

Name Type Description Default
key dict

key defining a single KacheryZone

required
Source code in src/spyglass/sharing/sharing_kachery.py
@staticmethod
def set_zone(key: dict):
    """Set the kachery zone based on the key to KacheryZone

    Parameters
    ----------
    key : dict
        key defining a single KacheryZone

    """
    try:
        kachery_zone_name, kachery_cloud_dir = (KacheryZone & key).fetch1(
            "kachery_zone_name", "kachery_cloud_dir"
        )
    except DataJointError:
        raise Exception(
            f"{key} does not correspond to a single entry in KacheryZone."
        )
        return None
    # set the new zone and cloud directory
    os.environ[kachery_zone_envar] = kachery_zone_name
    os.environ[kachery_cloud_dir_envar] = kachery_cloud_dir

reset_zone() staticmethod

Resets the kachery zone environment variable to the default values.

Source code in src/spyglass/sharing/sharing_kachery.py
@staticmethod
def reset_zone():
    """Resets the kachery zone environment variable to the default values."""
    if default_kachery_zone is not None:
        os.environ[kachery_zone_envar] = default_kachery_zone
    if default_kachery_cloud_dir is not None:
        os.environ[kachery_cloud_dir_envar] = default_kachery_cloud_dir

set_resource_url(key) staticmethod

Sets the KACHERY_RESOURCE_URL based on the key corresponding to a single Kachery Zone

Parameters:

Name Type Description Default
key dict

key to retrieve a single kachery zone

required
Source code in src/spyglass/sharing/sharing_kachery.py
@staticmethod
def set_resource_url(key: dict):
    """Sets the KACHERY_RESOURCE_URL based on the key corresponding to a
    single Kachery Zone

    Parameters
    ----------
    key : dict
        key to retrieve a single kachery zone
    """
    try:
        kachery_zone_name, kachery_proxy = (KacheryZone & key).fetch1(
            "kachery_zone_name", "kachery_proxy"
        )
    except DataJointError:
        raise Exception(
            f"{key} does not correspond to a single entry in KacheryZone."
        )
    # set the new zone and cloud directory
    os.environ[kachery_zone_envar] = kachery_zone_name
    os.environ[kachery_resource_url_envar] = (
        kachery_proxy + "/r/" + kachery_zone_name
    )

reset_resource_url() staticmethod

Resets the KACHERY_RESOURCE_URL to the default value.

Source code in src/spyglass/sharing/sharing_kachery.py
@staticmethod
def reset_resource_url():
    """Resets the KACHERY_RESOURCE_URL to the default value."""
    KacheryZone.reset_zone()
    if default_kachery_resource_url is not None:
        os.environ[kachery_resource_url_envar] = (
            default_kachery_resource_url
        )

AnalysisNwbfileKachery

Bases: SpyglassMixin, Computed

Source code in src/spyglass/sharing/sharing_kachery.py
@schema
class AnalysisNwbfileKachery(SpyglassMixin, dj.Computed):
    definition = """
    -> AnalysisNwbfileKacherySelection
    ---
    analysis_file_uri='': varchar(200)  # the uri of the file
    """

    class LinkedFile(SpyglassMixin, dj.Part):
        definition = """
        -> AnalysisNwbfileKachery
        linked_file_rel_path: varchar(200) # the path for the linked file relative to the SPYGLASS_BASE_DIR environment variable
        ---
        linked_file_uri='': varchar(200) # the uri for the linked file
        """

    def make(self, key):
        """Populate with the uri of the analysis file"""
        # note that we're assuming that the user has initialized a kachery-cloud
        # client with kachery-cloud-init. Uncomment the line below once we are
        # sharing linked files as well.

        # linked_key = copy.deepcopy(key)

        logger.info(f'Linking {key["analysis_file_name"]} in kachery-cloud...')
        # set the kachery zone

        KacheryZone.set_zone(key)

        key["analysis_file_uri"] = kcl.link_file(
            AnalysisNwbfile().get_abs_path(key["analysis_file_name"])
        )
        logger.info(
            os.environ[kachery_zone_envar], os.environ[kachery_cloud_dir_envar]
        )
        logger.info(AnalysisNwbfile().get_abs_path(key["analysis_file_name"]))
        logger.info(kcl.load_file(key["analysis_file_uri"]))
        self.insert1(key)

        # we also need to insert any linked files
        # TODO: change this to automatically detect all linked files
        # self.LinkedFile.insert1(key)

        # reset the Kachery zone and cloud_dir to the defaults
        KacheryZone.reset_zone()

    @staticmethod
    def download_file(
        analysis_file_name: str, permit_fail: bool = False
    ) -> bool:
        """Download the specified analysis file and associated linked files
        from kachery-cloud if possible

        Parameters
        ----------
        analysis_file_name : str
            The name of the analysis file

        Returns
        ----------
        is_success : bool
            True if the file was successfully downloaded, False otherwise
        """
        fetched_list = (
            AnalysisNwbfileKachery & {"analysis_file_name": analysis_file_name}
        ).fetch("analysis_file_uri", "kachery_zone_name")
        downloaded = False
        for uri, kachery_zone_name in zip(fetched_list[0], fetched_list[1]):
            if len(uri) == 0:
                return False
            logger.info("uri:", uri)
            if kachery_download_file(
                uri=uri,
                dest=AnalysisNwbfile.get_abs_path(analysis_file_name),
                kachery_zone_name=kachery_zone_name,
            ):
                downloaded = True
                # now download the linked file(s)
                linked_files = (
                    AnalysisNwbfileKachery.LinkedFile
                    & {"analysis_file_name": analysis_file_name}
                ).fetch(as_dict=True)
                for file in linked_files:
                    uri = file["linked_file_uri"]
                    logger.info(f"attempting to download linked file uri {uri}")
                    linked_file_path = (
                        os.environ["SPYGLASS_BASE_DIR"]
                        + file["linked_file_rel_path"]
                    )
                    if not kachery_download_file(
                        uri=uri,
                        dest=linked_file_path,
                        kachery_zone_name=kachery_zone_name,
                    ):
                        raise Exception(
                            f"Linked file {linked_file_path} cannot be downloaded"
                        )
        if not downloaded and not permit_fail:
            raise Exception(f"{analysis_file_name} cannot be downloaded")

        return downloaded

make(key)

Populate with the uri of the analysis file

Source code in src/spyglass/sharing/sharing_kachery.py
def make(self, key):
    """Populate with the uri of the analysis file"""
    # note that we're assuming that the user has initialized a kachery-cloud
    # client with kachery-cloud-init. Uncomment the line below once we are
    # sharing linked files as well.

    # linked_key = copy.deepcopy(key)

    logger.info(f'Linking {key["analysis_file_name"]} in kachery-cloud...')
    # set the kachery zone

    KacheryZone.set_zone(key)

    key["analysis_file_uri"] = kcl.link_file(
        AnalysisNwbfile().get_abs_path(key["analysis_file_name"])
    )
    logger.info(
        os.environ[kachery_zone_envar], os.environ[kachery_cloud_dir_envar]
    )
    logger.info(AnalysisNwbfile().get_abs_path(key["analysis_file_name"]))
    logger.info(kcl.load_file(key["analysis_file_uri"]))
    self.insert1(key)

    # we also need to insert any linked files
    # TODO: change this to automatically detect all linked files
    # self.LinkedFile.insert1(key)

    # reset the Kachery zone and cloud_dir to the defaults
    KacheryZone.reset_zone()

download_file(analysis_file_name, permit_fail=False) staticmethod

Download the specified analysis file and associated linked files from kachery-cloud if possible

Parameters:

Name Type Description Default
analysis_file_name str

The name of the analysis file

required

Returns:

Name Type Description
is_success bool

True if the file was successfully downloaded, False otherwise

Source code in src/spyglass/sharing/sharing_kachery.py
@staticmethod
def download_file(
    analysis_file_name: str, permit_fail: bool = False
) -> bool:
    """Download the specified analysis file and associated linked files
    from kachery-cloud if possible

    Parameters
    ----------
    analysis_file_name : str
        The name of the analysis file

    Returns
    ----------
    is_success : bool
        True if the file was successfully downloaded, False otherwise
    """
    fetched_list = (
        AnalysisNwbfileKachery & {"analysis_file_name": analysis_file_name}
    ).fetch("analysis_file_uri", "kachery_zone_name")
    downloaded = False
    for uri, kachery_zone_name in zip(fetched_list[0], fetched_list[1]):
        if len(uri) == 0:
            return False
        logger.info("uri:", uri)
        if kachery_download_file(
            uri=uri,
            dest=AnalysisNwbfile.get_abs_path(analysis_file_name),
            kachery_zone_name=kachery_zone_name,
        ):
            downloaded = True
            # now download the linked file(s)
            linked_files = (
                AnalysisNwbfileKachery.LinkedFile
                & {"analysis_file_name": analysis_file_name}
            ).fetch(as_dict=True)
            for file in linked_files:
                uri = file["linked_file_uri"]
                logger.info(f"attempting to download linked file uri {uri}")
                linked_file_path = (
                    os.environ["SPYGLASS_BASE_DIR"]
                    + file["linked_file_rel_path"]
                )
                if not kachery_download_file(
                    uri=uri,
                    dest=linked_file_path,
                    kachery_zone_name=kachery_zone_name,
                ):
                    raise Exception(
                        f"Linked file {linked_file_path} cannot be downloaded"
                    )
    if not downloaded and not permit_fail:
        raise Exception(f"{analysis_file_name} cannot be downloaded")

    return downloaded

share_data_to_kachery(restriction={}, table_list=[], zone_name=None)

Share data to kachery

Parameters:

Name Type Description Default
restriction dict

restriction to select what data should be shared from table, by default {}

{}
table_list list

List of tables to share data from, by default []

[]
zone_name str

What kachery zone to share the data to, by default zone in spyglass.settings.config, which looks for KACHERY_ZONE environmental variable, but defaults to 'franklab.default'

None

Raises:

Type Description
ValueError

Does not allow sharing of all data in table

Source code in src/spyglass/sharing/sharing_kachery.py
def share_data_to_kachery(
    restriction={},
    table_list=[],
    zone_name=None,
):
    """Share data to kachery

    Parameters
    ----------
    restriction : dict, optional
        restriction to select what data should be shared from table, by default {}
    table_list : list, optional
        List of tables to share data from, by default []
    zone_name : str, optional
        What kachery zone to share the data to, by default zone in spyglass.settings.config,
        which looks for `KACHERY_ZONE` environmental variable, but defaults to
        'franklab.default'

    Raises
    ------
    ValueError
        Does not allow sharing of all data in table
    """
    if not zone_name:
        zone_name = config["KACHERY_ZONE"]
    kachery_selection_key = {"kachery_zone_name": zone_name}
    if not restriction:
        raise ValueError("Must provide a restriction to the table")
    selection_inserts = []
    for table in table_list:
        analysis_file_list = (table & restriction).fetch("analysis_file_name")
        for file in analysis_file_list:  # Add all analysis to shared list
            kachery_selection_key["analysis_file_name"] = file
            selection_inserts.append(kachery_selection_key)
    AnalysisNwbfileKacherySelection.insert(
        selection_inserts, skip_duplicates=True
    )
    AnalysisNwbfileKachery.populate()