"""A module for ad-hoc dataset modification and registration.
Unlike the DataHandler class in oneibl.data_handlers, the Patcher class allows one to fully remove
datasets (delete them from the database and repositories), and to overwrite datasets on both the
main repositories and the local servers. Additionally the Patcher can handle datasets from
multiple sessions at once.
Examples
--------
Delete a dataset from Alyx and all associated repositories.
>>> dataset_id = 'f4aafe6c-a7ab-4390-82cd-2c0e245322a5'
>>> task_ids, files_by_repo = IBLGlobusPatcher(AlyxClient(), 'admin').delete_dataset(dataset_id)
Patch some local datasets using Globus
>>> from one.api import ONE
>>> patcher = GlobusPatcher('admin', ONE(), label='UCLA audio times patch')
>>> responses = patcher.patch_datasets(file_paths) # register the new datasets to Alyx
>>> patcher.launch_transfers(local_servers=True) # transfer to all remote repositories
"""
import abc
import ftplib
from pathlib import Path, PurePosixPath, WindowsPath
from collections import defaultdict
from itertools import starmap
from subprocess import Popen, PIPE, STDOUT
import subprocess
import logging
from getpass import getpass
import shutil
import globus_sdk
import iblutil.io.params as iopar
from one.alf.files import get_session_path, add_uuid_string
from one.alf.spec import is_uuid_string, is_uuid
from one import params
from one.webclient import AlyxClient
from one.converters import path_from_dataset
from one.remote import globus
from one.remote.aws import url2uri
from one.util import ensure_list
from ibllib.oneibl.registration import register_dataset
_logger = logging.getLogger(__name__)
FLATIRON_HOST = 'ibl.flatironinstitute.org'
FLATIRON_PORT = 61022
FLATIRON_USER = 'datauser'
FLATIRON_MOUNT = '/mnt/ibl'
FTP_HOST = 'test.alyx.internationalbrainlab.org'
FTP_PORT = 21
DMZ_REPOSITORY = 'ibl_patcher' # in alyx, the repository name containing the patched filerecords
SDSC_ROOT_PATH = PurePosixPath('/mnt/ibl')
SDSC_PATCH_PATH = PurePosixPath('/home/datauser/temp')
def _run_command(cmd, dry=True):
_logger.info(cmd)
if dry:
return 0, '', ''
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
info, error = p.communicate()
if p.returncode != 0:
_logger.error(error)
raise RuntimeError(error)
return p.returncode, info, error
[docs]
def sdsc_globus_path_from_dataset(dset):
"""
:param dset: dset dictionary or list of dictionaries from ALyx rest endpoint
Returns SDSC globus file path from a dset record or a list of dsets records from REST
"""
return path_from_dataset(dset, root_path=PurePosixPath('/'), repository=None, uuid=True)
[docs]
def sdsc_path_from_dataset(dset, root_path=SDSC_ROOT_PATH):
"""
Returns sdsc file path from a dset record or a list of dsets records from REST
:param dset: dset dictionary or list of dictionaries from ALyx rest endpoint
:param root_path: (optional) the prefix path such as one download directory or SDSC root
"""
return path_from_dataset(dset, root_path=root_path, uuid=True)
[docs]
def globus_path_from_dataset(dset, repository=None, uuid=False):
"""
Returns local one file path from a dset record or a list of dsets records from REST
:param dset: dset dictionary or list of dictionaries from ALyx rest endpoint
:param repository: (optional) repository name of the file record (if None, will take
the first filerecord with a URL)
"""
return path_from_dataset(dset, root_path=PurePosixPath('/'), repository=repository, uuid=uuid)
[docs]
class Patcher(abc.ABC):
def __init__(self, one=None):
assert one
self.one = one
def _patch_dataset(self, path, dset_id=None, dry=False, ftp=False):
"""
This private methods gets the dataset information from alyx, computes the local
and remote paths and initiates the file copy
"""
path = Path(path)
if dset_id is None:
dset_id = path.name.split('.')[-2]
if not is_uuid_string(dset_id):
dset_id = None
assert dset_id
assert is_uuid_string(dset_id)
assert path.exists()
dset = self.one.alyx.rest('datasets', 'read', id=dset_id)
fr = next(fr for fr in dset['file_records'] if 'flatiron' in fr['data_repository'])
remote_path = Path(fr['data_repository_path']).joinpath(fr['relative_path'])
remote_path = add_uuid_string(remote_path, dset_id).as_posix()
if remote_path.startswith('/'):
full_remote_path = PurePosixPath(FLATIRON_MOUNT + remote_path)
else:
full_remote_path = PurePosixPath(FLATIRON_MOUNT, remote_path)
if isinstance(path, WindowsPath) and not ftp:
# On Windows replace drive map with Globus uri, e.g. C:/ -> /~/C/
path = globus.as_globus_path(path)
status = self._scp(path, full_remote_path, dry=dry)[0]
return status
[docs]
def register_dataset(self, file_list, **kwargs):
"""
Registers a set of files belonging to a session only on the server
:param file_list: (list of pathlib.Path)
:param created_by: (string) name of user in Alyx (defaults to 'root')
:param repository: optional: (string) name of the server repository in Alyx
:param versions: optional (list of strings): versions tags (defaults to ibllib version)
:param dry: (bool) False by default
:return:
"""
return register_dataset(file_list, one=self.one, server_only=True, exists=True, **kwargs)
[docs]
def register_datasets(self, file_list, **kwargs):
"""
Same as register_dataset but works with files belonging to different sessions
"""
register_dict = {}
# creates a dictionary of sessions with one file list per session
for f in file_list:
session_path = get_session_path(f)
label = '_'.join(session_path.parts[-3:])
if label in register_dict:
register_dict[label]['files'].append(f)
else:
register_dict[label] = {'session_path': session_path, 'files': [f]}
responses = []
nses = len(register_dict)
for i, label in enumerate(register_dict):
_files = register_dict[label]['files']
_logger.info(f"{i + 1}/{nses} {label}, registering {len(_files)} files")
responses.append(self.register_dataset(_files, **kwargs))
return responses
[docs]
def patch_dataset(self, file_list, dry=False, ftp=False, **kwargs):
"""
Creates a new dataset on FlatIron and uploads it from arbitrary location.
Rules for creation/patching are the same that apply for registration via Alyx
as this uses the registration endpoint to get the dataset.
An existing file (same session and path relative to session) will be patched.
:param path: full file path. Must be within an ALF session folder (subject/date/number)
can also be a list of full file paths belonging to the same session.
:param server_repository: Alyx server repository name
:param created_by: alyx username for the dataset (optional, defaults to root)
:param ftp: flag for case when using ftppatcher. Don't adjust windows path in
_patch_dataset when ftp=True
:return: the registrations response, a list of dataset records
"""
# first register the file
if not isinstance(file_list, list):
file_list = [Path(file_list)]
assert len(set([get_session_path(f) for f in file_list])) == 1
assert all([Path(f).exists() for f in file_list])
response = ensure_list(self.register_dataset(file_list, dry=dry, **kwargs))
if dry:
return
# from the dataset info, set flatIron flag to exists=True
for p, d in zip(file_list, response):
self._patch_dataset(p, dset_id=d['id'], dry=dry, ftp=ftp)
return response
[docs]
def patch_datasets(self, file_list, **kwargs):
"""
Same as create_dataset method but works with several sessions
"""
register_dict = {}
# creates a dictionary of sessions with one file list per session
for f in file_list:
session_path = get_session_path(f)
label = '_'.join(session_path.parts[-3:])
if label in register_dict:
register_dict[label]['files'].append(f)
else:
register_dict[label] = {'session_path': session_path, 'files': [f]}
responses = []
nses = len(register_dict)
for i, label in enumerate(register_dict):
_files = register_dict[label]['files']
_logger.info(f'{i + 1}/{nses} {label}, registering {len(_files)} files')
responses.extend(self.patch_dataset(_files, **kwargs))
return responses
@abc.abstractmethod
def _scp(self, *args, **kwargs):
pass
@abc.abstractmethod
def _rm(self, *args, **kwargs):
pass
[docs]
class GlobusPatcher(Patcher, globus.Globus):
"""
Requires GLOBUS keys access
"""
def __init__(self, client_name='default', one=None, label='ibllib patch'):
assert one and not one.offline
Patcher.__init__(self, one=one)
globus.Globus.__init__(self, client_name)
self.label = label
# get a dictionary of data repositories from Alyx (with globus ids)
self.fetch_endpoints_from_alyx(one.alyx)
flatiron_id = self.endpoints['flatiron_cortexlab']['id']
if 'flatiron' not in self.endpoints:
self.add_endpoint(flatiron_id, 'flatiron', root_path='/')
self.endpoints['flatiron'] = self.endpoints['flatiron_cortexlab']
# transfers/delete from the current computer to the flatiron: mandatory and executed first
local_id = self.endpoints['local']['id']
self.globus_transfer = globus_sdk.TransferData(
self.client, local_id, flatiron_id, verify_checksum=True, sync_level='checksum', label=label)
self.globus_delete = globus_sdk.DeleteData(self.client, flatiron_id, label=label)
# transfers/delete from flatiron to optional third parties to synchronize / delete
self.globus_transfers_locals = {}
self.globus_deletes_locals = {}
super().__init__(one=one)
def _scp(self, local_path, remote_path, dry=True):
remote_path = PurePosixPath('/').joinpath(
remote_path.relative_to(PurePosixPath(FLATIRON_MOUNT))
)
_logger.info(f"Globus copy {local_path} to {remote_path}")
local_path = globus.as_globus_path(local_path)
if not dry:
if isinstance(self.globus_transfer, globus_sdk.TransferData):
self.globus_transfer.add_item(local_path, remote_path.as_posix())
else:
self.globus_transfer.path_src.append(local_path)
self.globus_transfer.path_dest.append(remote_path.as_posix())
return 0, ''
def _rm(self, flatiron_path, dry=True):
flatiron_path = Path('/').joinpath(flatiron_path.relative_to(Path(FLATIRON_MOUNT)))
_logger.info(f'Globus del {flatiron_path}')
if not dry:
if isinstance(self.globus_delete, globus_sdk.DeleteData):
self.globus_delete.add_item(flatiron_path.as_posix())
else:
self.globus_delete.path.append(flatiron_path.as_posix())
return 0, ''
[docs]
def patch_datasets(self, file_list, **kwargs):
"""
Calls the super method that registers and updates the current computer to Python transfer
Then, creates individual transfer items for each local server so that after the
update on Flatiron, local server files are also updated
:param file_list:
:param kwargs:
:return:
"""
responses = super().patch_datasets(file_list, **kwargs)
for dset in responses:
# get the flatiron path
fr = next(fr for fr in dset['file_records'] if 'flatiron' in fr['data_repository'])
relative_path = add_uuid_string(fr['relative_path'], dset['id']).as_posix()
flatiron_path = self.to_address(relative_path, fr['data_repository'])
# loop over the remaining repositories (local servers) and create a transfer
# from flatiron to the local server
for fr in dset['file_records']:
if fr['data_repository'] == DMZ_REPOSITORY:
continue
if fr['data_repository'] not in self.endpoints:
continue
repo_gid = self.endpoints[fr['data_repository']]['id']
flatiron_id = self.endpoints['flatiron']['id']
if repo_gid == flatiron_id:
continue
# if there is no transfer already created, initialize it
if repo_gid not in self.globus_transfers_locals:
self.globus_transfers_locals[repo_gid] = globus_sdk.TransferData(
self.client, flatiron_id, repo_gid, verify_checksum=True,
sync_level='checksum', label=f"{self.label} on {fr['data_repository']}")
# get the local server path and create the transfer item
local_server_path = self.to_address(fr['relative_path'], fr['data_repository'])
self.globus_transfers_locals[repo_gid].add_item(flatiron_path, local_server_path)
return responses
[docs]
def launch_transfers(self, local_servers=False):
"""
patcher.launch_transfers()
Launches the globus transfer and delete from the local patch computer to the flat-rion
:param: local_servers (False): if True, sync the local servers after the main transfer
:return: None
"""
gtc = self.client
def _wait_for_task(resp):
# patcher.transfer_client.get_task(task_id='364fbdd2-4deb-11eb-8ffb-0a34088e79f9')
# on a good status:
# Out[22]: TransferResponse({'bytes_checksummed': 377736912, 'bytes_transferred': 3011090432, 'canceled_by_admin': None, 'canceled_by_admin_message': None, 'command': 'API 0.10', 'completion_time': None, 'deadline': '2021-01-06T18:10:05+00:00', 'delete_destination_extra': False, 'destination_endpoint': 'simonsfoundation#ibl', 'destination_endpoint_display_name': 'IBL Flatiron SDSC Data', 'destination_endpoint_id': 'ab2d064c-413d-11eb-b188-0ee0d5d9299f', 'directories': 0, 'effective_bytes_per_second': 873268, 'encrypt_data': False, 'fatal_error': None, 'faults': 6, 'files': 186, 'files_skipped': 12, 'files_transferred': 76, 'history_deleted': False, 'is_ok': True, 'is_paused': False, 'key': 'active,2021-01-03T17:52:34.427087', 'label': '3B analog sync patch', 'nice_status': 'OK', 'nice_status_details': None, 'nice_status_expires_in': -1, 'nice_status_short_description': 'OK', 'owner_id': 'e633663a-8561-4a5d-ac92-f198d43b14dc', 'preserve_timestamp': False, 'recursive_symlinks': 'ignore', 'request_time': '2021-01-03T17:52:34+00:00', 'source_endpoint': 'internationalbrainlab#916c2766-bd2a-11ea-8f22-0a21f750d19b', 'source_endpoint_display_name': 'olivier_laptop', 'source_endpoint_id': '916c2766-bd2a-11ea-8f22-0a21f750d19b', 'status': 'ACTIVE', 'subtasks_canceled': 0, 'subtasks_expired': 0, 'subtasks_failed': 0, 'subtasks_pending': 98, 'subtasks_retrying': 0, 'subtasks_succeeded': 274, 'subtasks_total': 372, 'symlinks': 0, 'sync_level': 3, 'task_id': '364fbdd2-4deb-11eb-8ffb-0a34088e79f9', 'type': 'TRANSFER', 'username': 'internationalbrainlab', 'verify_checksum': True}) # noqa
# on a checksum error
# Out[26]: TransferResponse({'bytes_checksummed': 377736912, 'bytes_transferred': 3715901232, 'canceled_by_admin': None, 'canceled_by_admin_message': None, 'command': 'API 0.10', 'completion_time': None, 'deadline': '2021-01-06T18:10:05+00:00', 'delete_destination_extra': False, 'destination_endpoint': 'simonsfoundation#ibl', 'destination_endpoint_display_name': 'IBL Flatiron SDSC Data', 'destination_endpoint_id': 'ab2d064c-413d-11eb-b188-0ee0d5d9299f', 'directories': 0, 'effective_bytes_per_second': 912410, 'encrypt_data': False, 'fatal_error': None, 'faults': 7, 'files': 186, 'files_skipped': 12, 'files_transferred': 102, 'history_deleted': False, 'is_ok': False, 'is_paused': False, 'key': 'active,2021-01-03T17:52:34.427087', 'label': '3B analog sync patch', 'nice_status': 'VERIFY_CHECKSUM', 'nice_status_details': None, 'nice_status_expires_in': -1, 'nice_status_short_description': 'checksum verification failed', 'owner_id': 'e633663a-8561-4a5d-ac92-f198d43b14dc', 'preserve_timestamp': False, 'recursive_symlinks': 'ignore', 'request_time': '2021-01-03T17:52:34+00:00', 'source_endpoint': 'internationalbrainlab#916c2766-bd2a-11ea-8f22-0a21f750d19b', 'source_endpoint_display_name': 'olivier_laptop', 'source_endpoint_id': '916c2766-bd2a-11ea-8f22-0a21f750d19b', 'status': 'ACTIVE', 'subtasks_canceled': 0, 'subtasks_expired': 0, 'subtasks_failed': 0, 'subtasks_pending': 72, 'subtasks_retrying': 0, 'subtasks_succeeded': 300, 'subtasks_total': 372, 'symlinks': 0, 'sync_level': 3, 'task_id': '364fbdd2-4deb-11eb-8ffb-0a34088e79f9', 'type': 'TRANSFER', 'username': 'internationalbrainlab', 'verify_checksum': True}) # noqa
# on a finished task
# Out[4]: TransferResponse({'bytes_checksummed': 377736912, 'bytes_transferred': 4998806664, 'canceled_by_admin': None, 'canceled_by_admin_message': None, 'command': 'API 0.10', 'completion_time': '2021-01-03T20:04:50+00:00', 'deadline': '2021-01-06T19:11:00+00:00', 'delete_destination_extra': False, 'destination_endpoint': 'simonsfoundation#ibl', 'destination_endpoint_display_name': 'IBL Flatiron SDSC Data', 'destination_endpoint_id': 'ab2d064c-413d-11eb-b188-0ee0d5d9299f', 'directories': 0, 'effective_bytes_per_second': 629960, 'encrypt_data': False, 'fatal_error': None, 'faults': 15, 'files': 186, 'files_skipped': 12, 'files_transferred': 174, 'history_deleted': False, 'is_ok': None, 'is_paused': False, 'key': 'complete,2021-01-03T20:04:49.540956', 'label': '3B analog sync patch', 'nice_status': None, 'nice_status_details': None, 'nice_status_expires_in': None, 'nice_status_short_description': None, 'owner_id': 'e633663a-8561-4a5d-ac92-f198d43b14dc', 'preserve_timestamp': False, 'recursive_symlinks': 'ignore', 'request_time': '2021-01-03T17:52:34+00:00', 'source_endpoint': 'internationalbrainlab#916c2766-bd2a-11ea-8f22-0a21f750d19b', 'source_endpoint_display_name': 'olivier_laptop', 'source_endpoint_id': '916c2766-bd2a-11ea-8f22-0a21f750d19b', 'status': 'SUCCEEDED', 'subtasks_canceled': 0, 'subtasks_expired': 0, 'subtasks_failed': 0, 'subtasks_pending': 0, 'subtasks_retrying': 0, 'subtasks_succeeded': 372, 'subtasks_total': 372, 'symlinks': 0, 'sync_level': 3, 'task_id': '364fbdd2-4deb-11eb-8ffb-0a34088e79f9', 'type': 'TRANSFER', 'username': 'internationalbrainlab', 'verify_checksum': True}) # noqa
# on an errored task
# Out[10]: TransferResponse({'bytes_checksummed': 0, 'bytes_transferred': 0, 'canceled_by_admin': None, 'canceled_by_admin_message': None, 'command': 'API 0.10', 'completion_time': '2021-01-03T17:39:00+00:00', 'deadline': '2021-01-04T17:37:34+00:00', 'delete_destination_extra': False, 'destination_endpoint': 'simonsfoundation#ibl', 'destination_endpoint_display_name': 'IBL Flatiron SDSC Data', 'destination_endpoint_id': 'ab2d064c-413d-11eb-b188-0ee0d5d9299f', 'directories': 0, 'effective_bytes_per_second': 0, 'encrypt_data': False, 'fatal_error': {'code': 'CANCELED', 'description': 'canceled'}, 'faults': 2, 'files': 6, 'files_skipped': 0, 'files_transferred': 0, 'history_deleted': False, 'is_ok': None, 'is_paused': False, 'key': 'complete,2021-01-03T17:38:59.697413', 'label': 'test 3B analog sync patch', 'nice_status': None, 'nice_status_details': None, 'nice_status_expires_in': None, 'nice_status_short_description': None, 'owner_id': 'e633663a-8561-4a5d-ac92-f198d43b14dc', 'preserve_timestamp': False, 'recursive_symlinks': 'ignore', 'request_time': '2021-01-03T17:37:34+00:00', 'source_endpoint': 'internationalbrainlab#916c2766-bd2a-11ea-8f22-0a21f750d19b', 'source_endpoint_display_name': 'olivier_laptop', 'source_endpoint_id': '916c2766-bd2a-11ea-8f22-0a21f750d19b', 'status': 'FAILED', 'subtasks_canceled': 6, 'subtasks_expired': 0, 'subtasks_failed': 0, 'subtasks_pending': 0, 'subtasks_retrying': 0, 'subtasks_succeeded': 6, 'subtasks_total': 12, 'symlinks': 0, 'sync_level': 3, 'task_id': '5706dd2c-4dea-11eb-8ffb-0a34088e79f9', 'type': 'TRANSFER', 'username': 'internationalbrainlab', 'verify_checksum': True}) # noqa
while True:
tinfo = gtc.get_task(task_id=resp['task_id'])
if tinfo and tinfo['completion_time'] is not None:
break
_ = gtc.task_wait(task_id=resp['task_id'], timeout=30)
if tinfo and tinfo['fatal_error'] is not None:
raise ConnectionError(f"Globus transfer failed \n {tinfo}")
# handles the transfers first
if len(self.globus_transfer['DATA']) > 0:
# launch the transfer
_wait_for_task(gtc.submit_transfer(self.globus_transfer))
# re-initialize the globus_transfer property
self.globus_transfer = globus_sdk.TransferData(
gtc,
self.globus_transfer['source_endpoint'],
self.globus_transfer['destination_endpoint'],
label=self.globus_transfer['label'],
verify_checksum=True, sync_level='checksum')
# do the same for deletes
if len(self.globus_delete['DATA']) > 0:
_wait_for_task(gtc.submit_delete(self.globus_delete))
self.globus_delete = globus_sdk.DeleteData(
gtc,
endpoint=self.globus_delete['endpoint'],
label=self.globus_delete['label'])
# launch the local transfers and local deletes
if local_servers:
self.launch_transfers_secondary()
[docs]
def launch_transfers_secondary(self):
"""
patcher.launch_transfer_secondary()
Launches the globus transfers from flatiron to third-party repositories (local servers)
This should run after the the main transfer from patch computer to the flatiron
:return: None
"""
for lt in self.globus_transfers_locals:
transfer = self.globus_transfers_locals[lt]
if len(transfer['DATA']) > 0:
self.client.submit_transfer(transfer)
for ld in self.globus_deletes_locals:
delete = self.globus_deletes_locals[ld]
if len(transfer['DATA']) > 0:
self.client.submit_delete(delete)
[docs]
class IBLGlobusPatcher(Patcher, globus.Globus):
"""This is a replacement for the GlobusPatcher class, utilizing the ONE Globus class.
The GlobusPatcher class is more complicated but has the advantage of being able to launch
transfers independently to registration, although it remains to be seen whether this is useful.
"""
def __init__(self, alyx=None, client_name='default'):
"""
Parameters
----------
alyx : one.webclient.AlyxClient
An instance of Alyx to use.
client_name : str, default='default'
The Globus client name.
"""
self.alyx = alyx or AlyxClient()
globus.Globus.__init__(client_name=client_name) # NB we don't init Patcher as we're not using ONE
[docs]
def delete_dataset(self, dataset, dry=False):
"""
Delete a dataset off Alyx and remove file record from all Globus repositories.
Parameters
----------
dataset : uuid.UUID, str, dict
The dataset record or ID to delete.
dry : bool
If true, dataset is not deleted and file paths that would be removed are returned.
Returns
-------
list of uuid.UUID
A list of Globus delete task IDs if dry is false.
dict of str
A map of data repository names and relative paths of the deleted files.
"""
if is_uuid(dataset):
did = dataset
dataset = self.alyx.rest('datasets', 'read', id=did)
else:
did = dataset['url'].split('/')[-1]
def is_aws(repository_name):
return repository_name.startswith('aws_')
files_by_repo = defaultdict(list) # str -> [pathlib.PurePosixPath]
s3_files = []
file_records = filter(lambda x: x['exists'], dataset['file_records'])
for record in file_records:
repo = self.repo_from_alyx(record['data_repository'], self.alyx)
# Handle S3 files
if not repo['globus_endpoint_id'] or repo['repository_type'] != 'Fileserver':
if is_aws(repo['name']):
s3_files.append(url2uri(record['data_url']))
files_by_repo[repo['name']].append(PurePosixPath(record['relative_path']))
else:
_logger.error('Unable to delete from %s', repo['name'])
else:
# Handle Globus files
if repo['name'] not in self.endpoints:
self.add_endpoint(repo['name'], alyx=self.alyx)
filepath = PurePosixPath(record['relative_path'])
if 'flatiron' in repo['name']:
filepath = add_uuid_string(filepath, did)
files_by_repo[repo['name']].append(filepath)
# Remove S3 files
if s3_files:
cmd = ['aws', 's3', 'rm', *s3_files, '--profile', 'ibladmin']
if dry:
cmd.append('--dryrun')
if _logger.level > logging.DEBUG:
log_function = _logger.error
cmd.append('--only-show-errors') # Suppress verbose output
else:
log_function = _logger.debug
cmd.append('--no-progress') # Suppress progress info, estimated time, etc.
_logger.debug(' '.join(cmd))
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
for line in iter(process.stdout.readline, b''):
log_function(line.decode().strip())
assert process.wait() == 0
if dry:
return [], files_by_repo
# Remove Globus files
globus_files_map = filter(lambda x: not is_aws(x[0]), files_by_repo.items())
task_ids = list(starmap(self.delete_data, map(reversed, globus_files_map)))
# Delete the dataset from Alyx
self.alyx.rest('datasets', 'delete', id=did)
return task_ids, files_by_repo
[docs]
class SSHPatcher(Patcher):
"""
Requires SSH keys access on the FlatIron
"""
def __init__(self, one=None):
res = _run_command(f"ssh -p {FLATIRON_PORT} {FLATIRON_USER}@{FLATIRON_HOST} ls")
if res[0] != 0:
raise PermissionError("Could not connect to the Flatiron via SSH. Check your RSA keys")
super().__init__(one=one)
def _scp(self, local_path, remote_path, dry=True):
cmd = f"ssh -p {FLATIRON_PORT} {FLATIRON_USER}@{FLATIRON_HOST}" \
f" mkdir -p {remote_path.parent}; "
cmd += f"scp -P {FLATIRON_PORT} {local_path} {FLATIRON_USER}@{FLATIRON_HOST}:{remote_path}"
return _run_command(cmd, dry=dry)
def _rm(self, flatiron_path, dry=True):
cmd = f"ssh -p {FLATIRON_PORT} {FLATIRON_USER}@{FLATIRON_HOST} rm {flatiron_path}"
return _run_command(cmd, dry=dry)
[docs]
class FTPPatcher(Patcher):
"""
This is used to register from anywhere without write access to FlatIron
"""
def __init__(self, one=None):
super().__init__(one=one)
alyx = self.one.alyx
if not getattr(alyx._par, 'FTP_DATA_SERVER_LOGIN', False):
alyx._par = self.setup(par=alyx._par, silent=alyx.silent)
login, pwd = (one.alyx._par.FTP_DATA_SERVER_LOGIN, one.alyx._par.FTP_DATA_SERVER_PWD)
self.ftp = ftplib.FTP_TLS(host=FTP_HOST, user=login, passwd=pwd)
# self.ftp.ssl_version = ssl.PROTOCOL_TLSv1
# self.ftp.auth()
self.ftp.prot_p()
self.ftp.login(login, pwd)
# pre-fetch the repositories so as not to query them for every file registered
self.repositories = self.one.alyx.rest("data-repository", "list")
[docs]
@staticmethod
def setup(par=None, silent=False):
"""
Set up (and save) FTP login parameters
:param par: A parameters object to modify, if None the default Webclient parameters are
loaded
:param silent: If true, the defaults are used with no user input prompt
:return: the modified parameters object
"""
DEFAULTS = {
"FTP_DATA_SERVER": "ftp://ibl.flatironinstitute.org",
"FTP_DATA_SERVER_LOGIN": "iblftp",
"FTP_DATA_SERVER_PWD": None
}
if par is None:
par = params.get(silent=silent)
par = iopar.as_dict(par)
if silent:
DEFAULTS.update(par)
par = DEFAULTS
else:
for k in DEFAULTS.keys():
cpar = par.get(k, DEFAULTS[k])
# Iterate through non-password pars; skip url if client url already provided
if 'PWD' not in k:
par[k] = input(f'Param {k}, current value is ["{cpar}"]:') or cpar
else:
prompt = f'Param {k} (leave empty to leave unchanged):'
par[k] = getpass(prompt) or cpar
# Get the client key
client = par.get('ALYX_URL', None)
client_key = params._key_from_url(client) if client else params.get_default_client()
# Save the parameters
params.save(par, client_key) # Client params
return iopar.from_dict(par)
[docs]
def create_dataset(self, path, created_by='root', dry=False, repository=DMZ_REPOSITORY,
**kwargs):
# overrides the superclass just to remove the server repository argument
response = super().patch_dataset(path, created_by=created_by, dry=dry,
repository=repository, ftp=True, **kwargs)
# need to patch the file records to be consistent
for ds in response:
frs = ds['file_records']
fr_server = next(filter(lambda fr: 'flatiron' in fr['data_repository'], frs))
fr_ftp = next(filter(lambda fr: fr['data_repository'] == DMZ_REPOSITORY and
fr['relative_path'] == fr_server['relative_path'], frs))
reposerver = next(filter(lambda rep: rep['name'] == fr_server['data_repository'],
self.repositories))
relative_path = str(PurePosixPath(reposerver['globus_path']).joinpath(
PurePosixPath(fr_ftp['relative_path'])))[1:]
# 1) if there was already a file, the registration created a duplicate
fr_2del = list(filter(lambda fr: fr['data_repository'] == DMZ_REPOSITORY and
fr['relative_path'] == relative_path, frs)) # NOQA
if len(fr_2del) == 1:
self.one.alyx.rest('files', 'delete', id=fr_2del[0]['id'])
# 2) the patch ftp file needs to be prepended with the server repository path
self.one.alyx.rest('files', 'partial_update', id=fr_ftp['id'],
data={'relative_path': relative_path, 'exists': True})
# 3) the server file is labeled as not existing
self.one.alyx.rest('files', 'partial_update', id=fr_server['id'],
data={'exists': False})
return response
def _scp(self, local_path, remote_path, dry=True):
# remote_path = '/mnt/ibl/zadorlab/Subjects/flowers/2018-07-13/001
remote_path = PurePosixPath('/').joinpath(
remote_path.relative_to(PurePosixPath(FLATIRON_MOUNT))
)
# local_path
self.mktree(remote_path.parent)
self.ftp.pwd()
_logger.info(f"FTP upload {local_path}")
with open(local_path, 'rb') as fid:
self.ftp.storbinary(f'STOR {local_path.name}', fid)
return 0, ''
[docs]
def mktree(self, remote_path):
""" Browse to the tree on the ftp server, making directories on the way"""
if str(remote_path) != '.':
try:
self.ftp.cwd(str(remote_path))
except ftplib.error_perm:
self.mktree(PurePosixPath(remote_path.parent))
self.ftp.mkd(str(remote_path))
self.ftp.cwd(str(remote_path))
def _rm(self, flatiron_path, dry=True):
raise PermissionError("This Patcher does not have admin permissions to remove data "
"from the FlatIron server. ")
[docs]
class SDSCPatcher(Patcher):
"""
This is used to patch data on the SDSC server
"""
def __init__(self, one=None):
assert one
super().__init__(one=one)
[docs]
def patch_datasets(self, file_list, **kwargs):
response = super().patch_datasets(file_list, **kwargs)
# TODO check the file records to see if they have local server ones
# If they do then need to remove file record and delete file from local server??
return response
def _scp(self, local_path, remote_path, dry=True):
_logger.info(f"Copy {local_path} to {remote_path}")
if not dry:
if not Path(remote_path).parent.exists():
Path(remote_path).parent.mkdir(exist_ok=True, parents=True)
shutil.copy(local_path, remote_path)
return 0, ''
def _rm(self, flatiron_path, dry=True):
raise PermissionError("This Patcher does not have admin permissions to remove data "
"from the FlatIron server")