Skip to content

settings.py

SpyglassConfig

Gets Spyglass dirs from dj.config or environment variables.

Uses SpyglassConfig.relative_dirs to (a) gather user settings from dj.config or os environment variables or defaults relative to base, in that order (b) set environment variables, and (c) make dirs that don't exist. NOTE: when passed a base_dir, it will ignore env vars to facilitate testing.

Source code in src/spyglass/settings.py
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
class SpyglassConfig:
    """Gets Spyglass dirs from dj.config or environment variables.

    Uses SpyglassConfig.relative_dirs to (a) gather user
    settings from dj.config or os environment variables or defaults relative to
    base, in that order (b) set environment variables, and (c) make dirs that
    don't exist. NOTE: when passed a base_dir, it will ignore env vars to
    facilitate testing.
    """

    def __init__(self, base_dir: str = None, **kwargs) -> None:
        """
        Initializes a new instance of the class.

        Parameters
        ----------
        base_dir (str)
            The base directory.

        Attributes
        ----------
        supplied_base_dir (str)
            The base directory passed to the class.
        config_defaults (dict)
            Default settings for the config.
        relative_dirs (dict)
            Relative dirs for each prefix (spyglass, kachery, dlc). Relative
            to respective base_dir. Created on init.
        dj_defaults (dict)
            Default settings for datajoint.
        env_defaults (dict)
            Default settings for environment variables.
        _config (dict)
            Cached config settings.
        _debug_mode (bool)
            True if debug_mode is set. Supports skipping known bugs in test env.
        _test_mode (bool)
            True if test_mode is set. Required for pytests to run without
            prompts.
        """
        self.supplied_base_dir = base_dir
        self._config = dict()
        self.config_defaults = dict(prepopulate=True)
        self._debug_mode = kwargs.get("debug_mode", False)
        self._test_mode = kwargs.get("test_mode", False)
        self._dlc_base = None
        self.load_failed = False

        self.relative_dirs = {
            # {PREFIX}_{KEY}_DIR, default dir relative to base_dir
            # NOTE: Adding new dir requires edit to HHMI hub
            "spyglass": {
                "raw": "raw",
                "analysis": "analysis",
                "recording": "recording",
                "sorting": "spikesorting",
                "waveforms": "waveforms",
                "temp": "tmp",
                "video": "video",
                "export": "export",
            },
            "kachery": {
                "cloud": ".kachery-cloud",
                "storage": "kachery_storage",
                "temp": "tmp",
            },
            "dlc": {
                "project": "projects",
                "video": "video",
                "output": "output",
            },
        }
        self.dj_defaults = {
            "database.host": kwargs.get("database_host", "lmf-db.cin.ucsf.edu"),
            "database.user": kwargs.get("database_user"),
            "database.port": kwargs.get("database_port", 3306),
            "database.use_tls": kwargs.get("database_use_tls", True),
            "filepath_checksum_size_limit": 1 * 1024**3,
            "enable_python_native_blobs": True,
        }
        self.env_defaults = {
            "FIGURL_CHANNEL": "franklab2",
            "DJ_SUPPORT_FILEPATH_MANAGEMENT": "TRUE",
            "KACHERY_CLOUD_EPHEMERAL": "TRUE",
            "HD5_USE_FILE_LOCKING": "FALSE",
        }

    def load_config(
        self,
        base_dir=None,
        force_reload=False,
        on_startup: bool = False,
        **kwargs,
    ) -> None:
        """
        Loads the configuration settings for the object.

        Order of precedence, where X is base, raw, analysis, etc.:
            1. SpyglassConfig(base_dir="string") for base dir only
            2. dj.config['custom']['{spyglass/kachery}_dirs']['X']
            3. os.environ['{SPYGLASS/KACHERY}_{X}_DIR']
            4. resolved_base_dir/X for non-base dirs

        Parameters
        ----------
        base_dir: str
            Optional. Default None. The base directory. If not provided, will
            use the env variable or existing config.
        force_reload: bool
            Optional. Default False. Default skip load if already completed.

        Raises
        ------
        ValueError
            If base_dir is not set in either dj.config or os.environ.

        Returns
        -------
        dict
            list of relative_dirs and other settings (e.g., prepopulate).
        """
        if not force_reload and self._config:
            return self._config

        dj_custom = dj.config.get("custom", {})
        dj_spyglass = dj_custom.get("spyglass_dirs", {})
        dj_kachery = dj_custom.get("kachery_dirs", {})
        dj_dlc = dj_custom.get("dlc_dirs", {})

        self._debug_mode = dj_custom.get("debug_mode", False)
        self._test_mode = kwargs.get("test_mode") or dj_custom.get(
            "test_mode", False
        )

        resolved_base = (
            base_dir
            or self.supplied_base_dir
            or dj_spyglass.get("base")
            or os.environ.get("SPYGLASS_BASE_DIR")
        )

        if resolved_base and not Path(resolved_base).exists():
            resolved_base = Path(resolved_base).expanduser()
        if not resolved_base or not Path(resolved_base).exists():
            if not on_startup:  # Only warn if not on startup
                logger.error(
                    f"Could not find SPYGLASS_BASE_DIR: {resolved_base}"
                    + "\n\tCheck dj.config['custom']['spyglass_dirs']['base']"
                    + "\n\tand os.environ['SPYGLASS_BASE_DIR']"
                )
            self.load_failed = True
            return

        self._dlc_base = (
            dj_dlc.get("base")
            or os.environ.get("DLC_BASE_DIR")
            or os.environ.get("DLC_PROJECT_PATH", "").split("projects")[0]
            or str(Path(resolved_base) / "deeplabcut")
        )
        Path(self._dlc_base).mkdir(exist_ok=True)

        config_dirs = {"SPYGLASS_BASE_DIR": str(resolved_base)}
        for prefix, dirs in self.relative_dirs.items():
            this_base = self._dlc_base if prefix == "dlc" else resolved_base
            for dir, dir_str in dirs.items():
                dir_env_fmt = self.dir_to_var(dir=dir, dir_type=prefix)

                env_loc = (  # Ignore env vars if base was passed to func
                    os.environ.get(dir_env_fmt)
                    if not self.supplied_base_dir
                    else None
                )

                source_config = (
                    dj_dlc
                    if prefix == "dlc"
                    else dj_kachery if prefix == "kachery" else dj_spyglass
                )
                dir_location = (
                    source_config.get(dir)
                    or env_loc
                    or str(Path(this_base) / dir_str)
                ).replace('"', "")

                config_dirs.update({dir_env_fmt: str(dir_location)})

        kachery_zone_dict = {
            "KACHERY_ZONE": (
                os.environ.get("KACHERY_ZONE")
                or dj.config.get("custom", {}).get("kachery_zone")
                or "franklab.default"
            )
        }

        loaded_env = self._load_env_vars()
        self._set_env_with_dict(
            {**config_dirs, **kachery_zone_dict, **loaded_env}
        )
        self._mkdirs_from_dict_vals(config_dirs)

        self._config = dict(
            debug_mode=self._debug_mode,
            test_mode=self._test_mode,
            **self.config_defaults,
            **config_dirs,
            **kachery_zone_dict,
            **loaded_env,
        )

        self._set_dj_config_stores()

        return self._config

    def _load_env_vars(self) -> dict:
        loaded_dict = {}
        for var, val in self.env_defaults.items():
            loaded_dict[var] = os.getenv(var, val)
        return loaded_dict

    def _set_env_with_dict(self, env_dict) -> None:
        # NOTE: Kept for backwards compatibility. Should be removed in future
        # for custom paths. Keep self.env_defaults.
        # SPYGLASS_BASE_DIR may be used for docker assembly of export
        for var, val in env_dict.items():
            os.environ[var] = str(val)

    def _mkdirs_from_dict_vals(self, dir_dict) -> None:
        if self._debug_mode:
            return
        for dir_str in dir_dict.values():
            Path(dir_str).mkdir(exist_ok=True)

    def _set_dj_config_stores(self, check_match=True, set_stores=True) -> None:
        """
        Checks dj.config['stores'] match resolved dirs. Ensures stores set.

        Parameters
        ----------
        dir_dict: dict
            Dictionary of resolved dirs.
        check_match: bool
            Optional. Default True. Check that dj.config['stores'] match
            resolved dirs.
        set_stores: bool
            Optional. Default True. Set dj.config['stores'] to resolved dirs.
        """

        mismatch_analysis = False
        mismatch_raw = False

        if check_match:
            dj_stores = dj.config.get("stores", {})
            store_r = dj_stores.get("raw", {}).get("location")
            store_a = dj_stores.get("analysis", {}).get("location")
            mismatch_raw = store_r and Path(store_r) != Path(self.raw_dir)
            mismatch_analysis = store_a and Path(store_a) != Path(
                self.analysis_dir
            )

        if set_stores:
            if mismatch_raw or mismatch_analysis:
                logger.warning(
                    "Setting config DJ stores to resolve mismatch.\n\t"
                    + f"raw     : {self.raw_dir}\n\t"
                    + f"analysis: {self.analysis_dir}"
                )
            dj.config.update(self._dj_stores)
            return

        if mismatch_raw or mismatch_analysis:
            raise ValueError(
                "dj.config['stores'] does not match resolved dirs."
                + f"\n\tdj.config['stores']: {dj_stores}"
                + f"\n\tResolved dirs: {self._dj_stores}"
            )

        return

    def dir_to_var(self, dir: str, dir_type: str = "spyglass") -> str:
        """Converts a dir string to an env variable name."""
        return f"{dir_type.upper()}_{dir.upper()}_DIR"

    def _generate_dj_config(
        self,
        base_dir: str = None,
        database_user: str = None,
        database_password: str = None,
        database_host: str = "lmf-db.cin.ucsf.edu",
        database_port: int = 3306,
        database_use_tls: bool = True,
        **kwargs,
    ) -> dict:
        """Generate a datajoint configuration file.

        Parameters
        ----------
        database_user : str, optional
            The database user. If not provided, resulting config will not
            specify.
        database_password : str, optional
            The database password. If not provided, resulting config will not
            specify.
        database_host : str, optional
            Default lmf-db.cin.ucsf.edu. MySQL host name.
        dapabase_port : int, optional
            Default 3306. Port number for MySQL server.
        database_use_tls : bool, optional
            Default True. Use TLS encryption.
        **kwargs: dict, optional
            Any other valid datajoint configuration parameters.
            Note: python will raise error for params with `.` in name.
        """

        if database_user:
            kwargs.update({"database.user": database_user})
        if database_password:
            kwargs.update({"database.password": database_password})

        kwargs.update(
            {
                "database.host": database_host,
                "database.port": database_port,
                "database.use_tls": database_use_tls,
            }
        )

        # `|` merges dictionaries
        return self.dj_defaults | self._dj_stores | self._dj_custom | kwargs

    def save_dj_config(
        self,
        save_method: str = "global",
        output_filename: str = None,
        base_dir=None,
        set_password=True,
        **kwargs,
    ) -> None:
        """Set the dj.config parameters, set password, and save config to file.

        Parameters
        ----------
        save_method : {'local', 'global', 'custom'}, optional
            The method to use to save the config. If either 'local' or 'global',
            datajoint builtins will be used to save.
        output_filename : str or Path, optional
            Default to datajoint global config. If save_method = 'custom', name
            of file to generate. Must end in either be either yaml or json.
        base_dir : str, optional
            The base directory. If not provided, will default to the env var
        set_password : bool, optional
            Default True. Set the database password.
        kwargs: dict, optional
            Any other valid datajoint configuration parameters, including
            database_user, database_password, database_host, database_port, etc.
            Note: python will raise error for params with `.` in name, so use
            underscores instead.
        """
        if base_dir:
            self.load_config(
                base_dir=base_dir, force_reload=True, on_startup=False
            )

        if output_filename:
            save_method = "custom"
            path = Path(output_filename).expanduser()  # Expand ~
            filepath = path if path.is_absolute() else path.absolute()
            filepath.parent.mkdir(exist_ok=True, parents=True)
            filepath = (
                filepath.with_suffix(".json")  # ensure suffix, default json
                if filepath.suffix not in [".json", ".yaml"]
                else filepath
            )
        elif save_method == "local":
            filepath = Path(".") / dj.settings.LOCALCONFIG
        elif save_method == "global":
            filepath = Path("~").expanduser() / dj.settings.GLOBALCONFIG
        else:
            raise ValueError(
                "For save_dj_config, either (a) save_method must be 'local' "
                + " or 'global' or (b) must provide custom output_filename."
            )

        dj.config.update(self._generate_dj_config(**kwargs))

        if set_password:
            try:
                dj.set_password()
            except OperationalError as e:
                warnings.warn(f"Database connection issues. Wrong pass?\n\t{e}")

        user_warn = (
            f"Replace existing file? {filepath.resolve()}\n\t"
            + "\n\t".join([f"{k}: {v}" for k, v in config.items()])
            + "\n"
        )

        if (
            not self.test_mode
            and filepath.exists()
            and dj.utils.user_choice(user_warn)[0] != "y"
        ):
            return

        if save_method == "global":
            dj.config.save_global(verbose=True)
            return

        if save_method == "local":
            dj.config.save_local(verbose=True)
            return

        with open(filepath, "w") as outfile:
            if filepath.suffix == ".yaml":
                yaml.dump(dj.config._conf, outfile, default_flow_style=False)
            else:
                json.dump(dj.config._conf, outfile, indent=2)
            logger.info(f"Saved config to {filepath}")

    @property
    def _dj_stores(self) -> dict:
        self.load_config()
        return {
            "stores": {
                "raw": {
                    "protocol": "file",
                    "location": self.raw_dir,
                    "stage": self.raw_dir,
                },
                "analysis": {
                    "protocol": "file",
                    "location": self.analysis_dir,
                    "stage": self.analysis_dir,
                },
            }
        }

    @property
    def _dj_custom(self) -> dict:
        self.load_config()
        return {
            "custom": {
                "debug_mode": str(self.debug_mode).lower(),
                "test_mode": str(self._test_mode).lower(),
                "spyglass_dirs": {
                    "base": self.base_dir,
                    "raw": self.raw_dir,
                    "analysis": self.analysis_dir,
                    "recording": self.recording_dir,
                    "sorting": self.sorting_dir,
                    "waveforms": self.waveforms_dir,
                    "temp": self.temp_dir,
                    "video": self.video_dir,
                    "export": self.export_dir,
                },
                "kachery_dirs": {
                    "cloud": self.config.get(
                        self.dir_to_var("cloud", "kachery")
                    ),
                    "storage": self.config.get(
                        self.dir_to_var("storage", "kachery")
                    ),
                    "temp": self.config.get(self.dir_to_var("temp", "kachery")),
                },
                "dlc_dirs": {
                    "base": self._dlc_base,
                    "project": self.dlc_project_dir,
                    "video": self.dlc_video_dir,
                    "output": self.dlc_output_dir,
                },
                "kachery_zone": "franklab.default",
            }
        }

    @property
    def config(self) -> dict:
        """Dictionary of config settings."""
        self.load_config()
        return self._config

    @property
    def base_dir(self) -> str:
        """Base directory as a string."""
        return self.config.get(self.dir_to_var("base"))

    @property
    def raw_dir(self) -> str:
        """Raw data directory as a string."""
        return self.config.get(self.dir_to_var("raw"))

    @property
    def analysis_dir(self) -> str:
        """Analysis directory as a string."""
        return self.config.get(self.dir_to_var("analysis"))

    @property
    def recording_dir(self) -> str:
        """Recording directory as a string."""
        return self.config.get(self.dir_to_var("recording"))

    @property
    def sorting_dir(self) -> str:
        """Sorting directory as a string."""
        return self.config.get(self.dir_to_var("sorting"))

    @property
    def waveforms_dir(self) -> str:
        """Waveforms directory as a string."""
        return self.config.get(self.dir_to_var("waveforms"))

    @property
    def temp_dir(self) -> str:
        """Temp directory as a string."""
        return self.config.get(self.dir_to_var("temp"))

    @property
    def video_dir(self) -> str:
        """Video directory as a string."""
        return self.config.get(self.dir_to_var("video"))

    @property
    def export_dir(self) -> str:
        """Export directory as a string."""
        return self.config.get(self.dir_to_var("export"))

    @property
    def debug_mode(self) -> bool:
        """Returns True if debug_mode is set.

        Supports skipping inserts for Dockerized development.
        """
        return self._debug_mode

    @property
    def test_mode(self) -> bool:
        """Returns True if test_mode is set.

        Required for pytests to run without prompts."""
        return self._test_mode

    @property
    def dlc_project_dir(self) -> str:
        """DLC project directory as a string."""
        return self.config.get(self.dir_to_var("project", "dlc"))

    @property
    def dlc_video_dir(self) -> str:
        """DLC video directory as a string."""
        return self.config.get(self.dir_to_var("video", "dlc"))

    @property
    def dlc_output_dir(self) -> str:
        """DLC output directory as a string."""
        return self.config.get(self.dir_to_var("output", "dlc"))

__init__(base_dir=None, **kwargs)

Initializes a new instance of the class.

Parameters:

Name Type Description Default
base_dir str

The base directory.

None

Attributes:

Name Type Description
supplied_base_dir (str)

The base directory passed to the class.

config_defaults (dict)

Default settings for the config.

relative_dirs (dict)

Relative dirs for each prefix (spyglass, kachery, dlc). Relative to respective base_dir. Created on init.

dj_defaults (dict)

Default settings for datajoint.

env_defaults (dict)

Default settings for environment variables.

_config (dict)

Cached config settings.

_debug_mode (bool)

True if debug_mode is set. Supports skipping known bugs in test env.

_test_mode (bool)

True if test_mode is set. Required for pytests to run without prompts.

Source code in src/spyglass/settings.py
def __init__(self, base_dir: str = None, **kwargs) -> None:
    """
    Initializes a new instance of the class.

    Parameters
    ----------
    base_dir (str)
        The base directory.

    Attributes
    ----------
    supplied_base_dir (str)
        The base directory passed to the class.
    config_defaults (dict)
        Default settings for the config.
    relative_dirs (dict)
        Relative dirs for each prefix (spyglass, kachery, dlc). Relative
        to respective base_dir. Created on init.
    dj_defaults (dict)
        Default settings for datajoint.
    env_defaults (dict)
        Default settings for environment variables.
    _config (dict)
        Cached config settings.
    _debug_mode (bool)
        True if debug_mode is set. Supports skipping known bugs in test env.
    _test_mode (bool)
        True if test_mode is set. Required for pytests to run without
        prompts.
    """
    self.supplied_base_dir = base_dir
    self._config = dict()
    self.config_defaults = dict(prepopulate=True)
    self._debug_mode = kwargs.get("debug_mode", False)
    self._test_mode = kwargs.get("test_mode", False)
    self._dlc_base = None
    self.load_failed = False

    self.relative_dirs = {
        # {PREFIX}_{KEY}_DIR, default dir relative to base_dir
        # NOTE: Adding new dir requires edit to HHMI hub
        "spyglass": {
            "raw": "raw",
            "analysis": "analysis",
            "recording": "recording",
            "sorting": "spikesorting",
            "waveforms": "waveforms",
            "temp": "tmp",
            "video": "video",
            "export": "export",
        },
        "kachery": {
            "cloud": ".kachery-cloud",
            "storage": "kachery_storage",
            "temp": "tmp",
        },
        "dlc": {
            "project": "projects",
            "video": "video",
            "output": "output",
        },
    }
    self.dj_defaults = {
        "database.host": kwargs.get("database_host", "lmf-db.cin.ucsf.edu"),
        "database.user": kwargs.get("database_user"),
        "database.port": kwargs.get("database_port", 3306),
        "database.use_tls": kwargs.get("database_use_tls", True),
        "filepath_checksum_size_limit": 1 * 1024**3,
        "enable_python_native_blobs": True,
    }
    self.env_defaults = {
        "FIGURL_CHANNEL": "franklab2",
        "DJ_SUPPORT_FILEPATH_MANAGEMENT": "TRUE",
        "KACHERY_CLOUD_EPHEMERAL": "TRUE",
        "HD5_USE_FILE_LOCKING": "FALSE",
    }

load_config(base_dir=None, force_reload=False, on_startup=False, **kwargs)

Loads the configuration settings for the object.

Order of precedence, where X is base, raw, analysis, etc.: 1. SpyglassConfig(base_dir="string") for base dir only 2. dj.config['custom']['{spyglass/kachery}_dirs']['X'] 3. os.environ['{SPYGLASS/KACHERY}_{X}_DIR'] 4. resolved_base_dir/X for non-base dirs

Parameters:

Name Type Description Default
base_dir

Optional. Default None. The base directory. If not provided, will use the env variable or existing config.

None
force_reload

Optional. Default False. Default skip load if already completed.

False

Raises:

Type Description
ValueError

If base_dir is not set in either dj.config or os.environ.

Returns:

Type Description
dict

list of relative_dirs and other settings (e.g., prepopulate).

Source code in src/spyglass/settings.py
def load_config(
    self,
    base_dir=None,
    force_reload=False,
    on_startup: bool = False,
    **kwargs,
) -> None:
    """
    Loads the configuration settings for the object.

    Order of precedence, where X is base, raw, analysis, etc.:
        1. SpyglassConfig(base_dir="string") for base dir only
        2. dj.config['custom']['{spyglass/kachery}_dirs']['X']
        3. os.environ['{SPYGLASS/KACHERY}_{X}_DIR']
        4. resolved_base_dir/X for non-base dirs

    Parameters
    ----------
    base_dir: str
        Optional. Default None. The base directory. If not provided, will
        use the env variable or existing config.
    force_reload: bool
        Optional. Default False. Default skip load if already completed.

    Raises
    ------
    ValueError
        If base_dir is not set in either dj.config or os.environ.

    Returns
    -------
    dict
        list of relative_dirs and other settings (e.g., prepopulate).
    """
    if not force_reload and self._config:
        return self._config

    dj_custom = dj.config.get("custom", {})
    dj_spyglass = dj_custom.get("spyglass_dirs", {})
    dj_kachery = dj_custom.get("kachery_dirs", {})
    dj_dlc = dj_custom.get("dlc_dirs", {})

    self._debug_mode = dj_custom.get("debug_mode", False)
    self._test_mode = kwargs.get("test_mode") or dj_custom.get(
        "test_mode", False
    )

    resolved_base = (
        base_dir
        or self.supplied_base_dir
        or dj_spyglass.get("base")
        or os.environ.get("SPYGLASS_BASE_DIR")
    )

    if resolved_base and not Path(resolved_base).exists():
        resolved_base = Path(resolved_base).expanduser()
    if not resolved_base or not Path(resolved_base).exists():
        if not on_startup:  # Only warn if not on startup
            logger.error(
                f"Could not find SPYGLASS_BASE_DIR: {resolved_base}"
                + "\n\tCheck dj.config['custom']['spyglass_dirs']['base']"
                + "\n\tand os.environ['SPYGLASS_BASE_DIR']"
            )
        self.load_failed = True
        return

    self._dlc_base = (
        dj_dlc.get("base")
        or os.environ.get("DLC_BASE_DIR")
        or os.environ.get("DLC_PROJECT_PATH", "").split("projects")[0]
        or str(Path(resolved_base) / "deeplabcut")
    )
    Path(self._dlc_base).mkdir(exist_ok=True)

    config_dirs = {"SPYGLASS_BASE_DIR": str(resolved_base)}
    for prefix, dirs in self.relative_dirs.items():
        this_base = self._dlc_base if prefix == "dlc" else resolved_base
        for dir, dir_str in dirs.items():
            dir_env_fmt = self.dir_to_var(dir=dir, dir_type=prefix)

            env_loc = (  # Ignore env vars if base was passed to func
                os.environ.get(dir_env_fmt)
                if not self.supplied_base_dir
                else None
            )

            source_config = (
                dj_dlc
                if prefix == "dlc"
                else dj_kachery if prefix == "kachery" else dj_spyglass
            )
            dir_location = (
                source_config.get(dir)
                or env_loc
                or str(Path(this_base) / dir_str)
            ).replace('"', "")

            config_dirs.update({dir_env_fmt: str(dir_location)})

    kachery_zone_dict = {
        "KACHERY_ZONE": (
            os.environ.get("KACHERY_ZONE")
            or dj.config.get("custom", {}).get("kachery_zone")
            or "franklab.default"
        )
    }

    loaded_env = self._load_env_vars()
    self._set_env_with_dict(
        {**config_dirs, **kachery_zone_dict, **loaded_env}
    )
    self._mkdirs_from_dict_vals(config_dirs)

    self._config = dict(
        debug_mode=self._debug_mode,
        test_mode=self._test_mode,
        **self.config_defaults,
        **config_dirs,
        **kachery_zone_dict,
        **loaded_env,
    )

    self._set_dj_config_stores()

    return self._config

dir_to_var(dir, dir_type='spyglass')

Converts a dir string to an env variable name.

Source code in src/spyglass/settings.py
def dir_to_var(self, dir: str, dir_type: str = "spyglass") -> str:
    """Converts a dir string to an env variable name."""
    return f"{dir_type.upper()}_{dir.upper()}_DIR"

save_dj_config(save_method='global', output_filename=None, base_dir=None, set_password=True, **kwargs)

Set the dj.config parameters, set password, and save config to file.

Parameters:

Name Type Description Default
save_method (local, 'global', custom)

The method to use to save the config. If either 'local' or 'global', datajoint builtins will be used to save.

'local'
output_filename str or Path

Default to datajoint global config. If save_method = 'custom', name of file to generate. Must end in either be either yaml or json.

None
base_dir str

The base directory. If not provided, will default to the env var

None
set_password bool

Default True. Set the database password.

True
kwargs

Any other valid datajoint configuration parameters, including database_user, database_password, database_host, database_port, etc. Note: python will raise error for params with . in name, so use underscores instead.

{}
Source code in src/spyglass/settings.py
def save_dj_config(
    self,
    save_method: str = "global",
    output_filename: str = None,
    base_dir=None,
    set_password=True,
    **kwargs,
) -> None:
    """Set the dj.config parameters, set password, and save config to file.

    Parameters
    ----------
    save_method : {'local', 'global', 'custom'}, optional
        The method to use to save the config. If either 'local' or 'global',
        datajoint builtins will be used to save.
    output_filename : str or Path, optional
        Default to datajoint global config. If save_method = 'custom', name
        of file to generate. Must end in either be either yaml or json.
    base_dir : str, optional
        The base directory. If not provided, will default to the env var
    set_password : bool, optional
        Default True. Set the database password.
    kwargs: dict, optional
        Any other valid datajoint configuration parameters, including
        database_user, database_password, database_host, database_port, etc.
        Note: python will raise error for params with `.` in name, so use
        underscores instead.
    """
    if base_dir:
        self.load_config(
            base_dir=base_dir, force_reload=True, on_startup=False
        )

    if output_filename:
        save_method = "custom"
        path = Path(output_filename).expanduser()  # Expand ~
        filepath = path if path.is_absolute() else path.absolute()
        filepath.parent.mkdir(exist_ok=True, parents=True)
        filepath = (
            filepath.with_suffix(".json")  # ensure suffix, default json
            if filepath.suffix not in [".json", ".yaml"]
            else filepath
        )
    elif save_method == "local":
        filepath = Path(".") / dj.settings.LOCALCONFIG
    elif save_method == "global":
        filepath = Path("~").expanduser() / dj.settings.GLOBALCONFIG
    else:
        raise ValueError(
            "For save_dj_config, either (a) save_method must be 'local' "
            + " or 'global' or (b) must provide custom output_filename."
        )

    dj.config.update(self._generate_dj_config(**kwargs))

    if set_password:
        try:
            dj.set_password()
        except OperationalError as e:
            warnings.warn(f"Database connection issues. Wrong pass?\n\t{e}")

    user_warn = (
        f"Replace existing file? {filepath.resolve()}\n\t"
        + "\n\t".join([f"{k}: {v}" for k, v in config.items()])
        + "\n"
    )

    if (
        not self.test_mode
        and filepath.exists()
        and dj.utils.user_choice(user_warn)[0] != "y"
    ):
        return

    if save_method == "global":
        dj.config.save_global(verbose=True)
        return

    if save_method == "local":
        dj.config.save_local(verbose=True)
        return

    with open(filepath, "w") as outfile:
        if filepath.suffix == ".yaml":
            yaml.dump(dj.config._conf, outfile, default_flow_style=False)
        else:
            json.dump(dj.config._conf, outfile, indent=2)
        logger.info(f"Saved config to {filepath}")

config: dict property

Dictionary of config settings.

base_dir: str property

Base directory as a string.

raw_dir: str property

Raw data directory as a string.

analysis_dir: str property

Analysis directory as a string.

recording_dir: str property

Recording directory as a string.

sorting_dir: str property

Sorting directory as a string.

waveforms_dir: str property

Waveforms directory as a string.

temp_dir: str property

Temp directory as a string.

video_dir: str property

Video directory as a string.

export_dir: str property

Export directory as a string.

debug_mode: bool property

Returns True if debug_mode is set.

Supports skipping inserts for Dockerized development.

test_mode: bool property

Returns True if test_mode is set.

Required for pytests to run without prompts.

dlc_project_dir: str property

DLC project directory as a string.

dlc_video_dir: str property

DLC video directory as a string.

dlc_output_dir: str property

DLC output directory as a string.