ibllib.pipes.ephys_preprocessing

(Deprecated) Electrophysiology data preprocessing tasks.

These tasks are part of the old pipeline. This module has been replaced by the ephys_tasks module and the dynamic pipeline.

Classes

EphysAudio	Compresses the microphone wav file in a lossless flac file
EphysCellsQc
EphysDLC	This task relies on a correctly installed dlc environment as per https://docs.google.com/document/d/1g0scP6_3EmaXCU4SsDNZWwDTaD9MG0es_grLA-d0gh0/edit#
EphysExtractionPipeline
EphysMtscomp
EphysPassive
EphysPostDLC	The post_dlc task takes dlc traces as input and computes useful quantities, as well as qc.
EphysPulses	Extract Pulses from raw electrophysiology data into numpy arrays Perform the probes synchronisation with nidq (3B) or main probe (3A)
EphysTrials
EphysVideoCompress
EphysVideoSyncQc
LaserTrialsLegacy	This is the legacy extractor for Guido's ephys optogenetic stimulation protocol.
RawEphysQC	Computes raw electrophysiology QC
SpikeSorting	(DEPRECATED) Pykilosort 2.5 pipeline

class EphysPulses(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

Extract Pulses from raw electrophysiology data into numpy arrays Perform the probes synchronisation with nidq (3B) or main probe (3A)

cpu = 2

io_charge = 30

priority = 90

level = 0

force = False

signature = {'input_files': [('*ap.meta', 'raw_ephys_data/probe*', True), ('*ap.ch', 'raw_ephys_data/probe*', False), ('*ap.*bin', 'raw_ephys_data/probe*', True), ('*nidq.meta', 'raw_ephys_data', True), ('*nidq.ch', 'raw_ephys_data', False), ('*nidq.*bin', 'raw_ephys_data', True)], 'output_files': [('_spikeglx_sync*.npy', 'raw_ephys_data*', True), ('_spikeglx_sync.polarities*.npy', 'raw_ephys_data*', True), ('_spikeglx_sync.times*.npy', 'raw_ephys_data*', True)]}

get_signatures(**kwargs)

Find the input and output signatures specific for local filesystem :return:

class RawEphysQC(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

Computes raw electrophysiology QC

cpu = 2

io_charge = 30

priority = 10

level = 0

force = False

signature = {'input_files': [('*ap.meta', 'raw_ephys_data/probe*', True), ('*lf.meta', 'raw_ephys_data/probe*', True), ('*lf.ch', 'raw_ephys_data/probe*', False), ('*lf.*bin', 'raw_ephys_data/probe*', True)], 'output_files': [('_iblqc_ephysChannels.apRMS.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysChannels.rawSpikeRates.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysChannels.labels.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysSpectralDensityLF.freqs.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysSpectralDensityLF.power.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysSpectralDensityAP.freqs.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysSpectralDensityAP.power.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysTimeRmsLF.rms.npy', 'raw_ephys_data/probe*', True), ('_iblqc_ephysTimeRmsLF.timestamps.npy', 'raw_ephys_data/probe*', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysAudio(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

Compresses the microphone wav file in a lossless flac file

cpu = 2

priority = 10

level = 0

force = False

signature = {'input_files': [('_iblrig_micData.raw.wav', 'raw_behavior_data', True)], 'output_files': [('_iblrig_micData.raw.flac', 'raw_behavior_data', True)]}

class SpikeSorting(*args, **kwargs)

Bases: Task

(DEPRECATED) Pykilosort 2.5 pipeline

gpu = 1

io_charge = 100

priority = 60

level = 1

force = True

job_size = 'large'

SHELL_SCRIPT = PosixPath('/home/runner/Documents/PYTHON/iblscripts/deploy/serverpc/kilosort2/run_pykilosort.sh')

SPIKE_SORTER_NAME = 'pykilosort'

PYKILOSORT_REPO = PosixPath('/home/runner/Documents/PYTHON/SPIKE_SORTING/pykilosort')

signature = {'input_files': [], 'output_files': []}

static spike_sorting_signature(pname=None)

static parse_version(v) → Version

Extracts and parses semantic version (major.minor.patch) from a version string.

Parameters:

v (str) – A version string containing a semantic version.

Returns:

The parsed semantic version number

Return type:

packaging.version.Version

Examples

>>> SpikeSorting.parse_version('ibl_1.2')
<Version('1.2')>
>>> SpikeSorting.parse_version('pykilosort_ibl_1.2.0-new')
<Version('1.2.0')>
>>> SpikeSorting.parse_version('ibl_0.2') < SpikeSorting.parse_version('pykilosort_v1')
True

setUp(probes=None)

Overwrite setup method to allow inputs and outputs to be only one probe

Parameters:

probes – list of probes e.g [‘probe00’]

Returns:

get_signatures(probes=None, **kwargs)

This transforms all wildcards in collection to exact match

Parameters:

probes

Returns:

class EphysVideoCompress(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

priority = 90

level = 0

force = False

job_size = 'large'

io_charge = 100

signature = {'input_files': [('_iblrig_*Camera.raw.*', 'raw_video_data', True)], 'output_files': [('_iblrig_*Camera.raw.mp4', 'raw_video_data', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysVideoSyncQc(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

priority = 40

level = 2

force = True

signature = {'input_files': [('_iblrig_*Camera.raw.mp4', 'raw_video_data', True), ('_iblrig_*Camera.timestamps.ssv', 'raw_video_data', False), ('_iblrig_*Camera.timestamps.npy', 'raw_video_data', False), ('_iblrig_*Camera.frameData.bin', 'raw_video_data', False), ('_iblrig_*Camera.GPIO.bin', 'raw_video_data', False), ('_iblrig_*Camera.frame_counter.bin', 'raw_video_data', False), ('_iblrig_taskData.raw.*', 'raw_behavior_data', True), ('_iblrig_taskSettings.raw.*', 'raw_behavior_data', True), ('_spikeglx_sync.channels.*', 'raw_ephys_data*', True), ('_spikeglx_sync.polarities.*', 'raw_ephys_data*', True), ('_spikeglx_sync.times.*', 'raw_ephys_data*', True), ('*wheel.position.npy', 'alf', False), ('*wheel.timestamps.npy', 'alf', False), ('*wiring.json', 'raw_ephys_data*', False), ('*.meta', 'raw_ephys_data*', True)], 'output_files': [('_ibl_*Camera.times.npy', 'alf', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysTrials(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

priority = 90

level = 1

force = False

signature = {'input_files': [('_iblrig_taskData.raw.*', 'raw_behavior_data', True), ('_iblrig_taskSettings.raw.*', 'raw_behavior_data', True), ('_spikeglx_sync.channels.*', 'raw_ephys_data*', True), ('_spikeglx_sync.polarities.*', 'raw_ephys_data*', True), ('_spikeglx_sync.times.*', 'raw_ephys_data*', True), ('_iblrig_encoderEvents.raw*', 'raw_behavior_data', True), ('_iblrig_encoderPositions.raw*', 'raw_behavior_data', True), ('*wiring.json', 'raw_ephys_data*', False), ('*.meta', 'raw_ephys_data*', True)], 'output_files': [('*trials.table.pqt', 'alf', True), ('*trials.goCueTrigger_times.npy', 'alf', True), ('*trials.intervals_bpod.npy', 'alf', True), ('*trials.stimOff_times.npy', 'alf', True), ('*trials.quiescencePeriod.npy', 'alf', True), ('*wheel.position.npy', 'alf', True), ('*wheel.timestamps.npy', 'alf', True), ('*wheelMoves.intervals.npy', 'alf', True), ('*wheelMoves.peakAmplitude.npy', 'alf', True)]}

extract_behaviour(save=True)

run_qc(trials_data=None, update=True, plot_qc=False)

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class LaserTrialsLegacy(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: EphysTrials

This is the legacy extractor for Guido’s ephys optogenetic stimulation protocol.

This is legacy because personal project extractors should be in a separate repository.

extract_behaviour()

class EphysCellsQc(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

priority = 90

level = 3

force = False

signature = {'input_files': [('spikes.times.npy', 'alf/probe*', True), ('spikes.clusters.npy', 'alf/probe*', True), ('spikes.amps.npy', 'alf/probe*', True), ('spikes.depths.npy', 'alf/probe*', True), ('clusters.channels.npy', 'alf/probe*', True)], 'output_files': [('clusters.metrics.pqt', 'alf/probe*', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysMtscomp(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

priority = 50

level = 0

force = False

signature = {'input_files': [('*ap.meta', 'raw_ephys_data/probe*', True), ('*ap.*bin', 'raw_ephys_data/probe*', True), ('*lf.meta', 'raw_ephys_data/probe*', False), ('*lf.*bin', 'raw_ephys_data/probe*', False), ('*nidq.meta', 'raw_ephys_data', True), ('*nidq.*bin', 'raw_ephys_data', True)], 'output_files': [('*ap.meta', 'raw_ephys_data/probe*', True), ('*ap.cbin', 'raw_ephys_data/probe*', False), ('*ap.ch', 'raw_ephys_data/probe*', True), ('*lf.meta', 'raw_ephys_data/probe*', False), ('*lf.cbin', 'raw_ephys_data/probe*', False), ('*lf.ch', 'raw_ephys_data/probe*', False), ('*nidq.meta', 'raw_ephys_data', True), ('*nidq.cbin', 'raw_ephys_data', False), ('*nidq.ch', 'raw_ephys_data', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysDLC(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

This task relies on a correctly installed dlc environment as per https://docs.google.com/document/d/1g0scP6_3EmaXCU4SsDNZWwDTaD9MG0es_grLA-d0gh0/edit#

If your environment is set up otherwise, make sure that you set the respective attributes: t = EphysDLC(session_path) t.dlcenv = Path(‘/path/to/your/dlcenv/bin/activate’) t.scripts = Path(‘/path/to/your/iblscripts/deploy/serverpc/dlc’)

gpu = 1

cpu = 4

io_charge = 100

level = 2

force = True

job_size = 'large'

dlcenv = PosixPath('/home/runner/Documents/PYTHON/envs/dlcenv/bin/activate')

scripts = PosixPath('/home/runner/Documents/PYTHON/iblscripts/deploy/serverpc/dlc')

signature = {'input_files': [('_iblrig_leftCamera.raw.mp4', 'raw_video_data', True), ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', True), ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', True)], 'output_files': [('_ibl_leftCamera.dlc.pqt', 'alf', True), ('_ibl_rightCamera.dlc.pqt', 'alf', True), ('_ibl_bodyCamera.dlc.pqt', 'alf', True), ('leftCamera.ROIMotionEnergy.npy', 'alf', True), ('rightCamera.ROIMotionEnergy.npy', 'alf', True), ('bodyCamera.ROIMotionEnergy.npy', 'alf', True), ('leftROIMotionEnergy.position.npy', 'alf', True), ('rightROIMotionEnergy.position.npy', 'alf', True), ('bodyROIMotionEnergy.position.npy', 'alf', True)]}

class EphysPostDLC(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

The post_dlc task takes dlc traces as input and computes useful quantities, as well as qc.

io_charge = 90

level = 3

force = True

signature = {'input_files': [('_ibl_leftCamera.dlc.pqt', 'alf', True), ('_ibl_bodyCamera.dlc.pqt', 'alf', True), ('_ibl_rightCamera.dlc.pqt', 'alf', True), ('_ibl_rightCamera.times.npy', 'alf', True), ('_ibl_leftCamera.times.npy', 'alf', True), ('_ibl_bodyCamera.times.npy', 'alf', True), ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', True), ('_iblrig_leftCamera.raw.mp4', 'raw_video_data', True), ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', True), ('rightROIMotionEnergy.position.npy', 'alf', False), ('leftROIMotionEnergy.position.npy', 'alf', False), ('bodyROIMotionEnergy.position.npy', 'alf', False), ('_ibl_trials.table.pqt', 'alf', True), ('_ibl_wheel.position.npy', 'alf', True), ('_ibl_wheel.timestamps.npy', 'alf', True)], 'output_files': [('_ibl_leftCamera.features.pqt', 'alf', True), ('_ibl_rightCamera.features.pqt', 'alf', True), ('licks.times.npy', 'alf', True)]}

class EphysPassive(session_path, parents=None, taskid=None, one=None, machine=None, clobber=True, location='server', **kwargs)

Bases: Task

cpu = 1

io_charge = 90

level = 1

force = False

signature = {'input_files': [('_iblrig_taskSettings.raw*', 'raw_behavior_data', True), ('_spikeglx_sync.channels.*', 'raw_ephys_data*', True), ('_spikeglx_sync.polarities.*', 'raw_ephys_data*', True), ('_spikeglx_sync.times.*', 'raw_ephys_data*', True), ('*.meta', 'raw_ephys_data*', True), ('*wiring.json', 'raw_ephys_data*', False), ('_iblrig_RFMapStim.raw*', 'raw_passive_data', True)], 'output_files': [('_ibl_passiveGabor.table.csv', 'alf', True), ('_ibl_passivePeriods.intervalsTable.csv', 'alf', True), ('_ibl_passiveRFM.times.npy', 'alf', True), ('_ibl_passiveStims.table.csv', 'alf', True)]}

get_signatures(**kwargs)

This is the default but should be overwritten for each task :return:

class EphysExtractionPipeline(session_path=None, **kwargs)

Bases: Pipeline

label = 'ibllib.pipes.ephys_preprocessing'