Commit 0c44a495 authored by Andrew Leonard's avatar Andrew Leonard
Browse files

Update KISClient.fetch to download from rucio server

parent 7a661ac1
......@@ -320,7 +320,7 @@ class KISClient(BaseClient):
@convert_row_to_table
def fetch(self, query_results: QueryResponseTable, *, downloader: parfive.Downloader,
path: os.PathLike = None, binary: bool = False, **kwargs):
path: os.PathLike = None, **kwargs):
"""
Fetch GridFS dataset metadata or binary files.
......@@ -338,22 +338,18 @@ class KISClient(BaseClient):
if not len(query_results):
return
if binary:
binfile = '/binary'
ext = 'fits'
else:
binfile = ''
ext = 'json'
for row in query_results:
inst = row['INSTRUMENT']
oid = row['links']['$oid']
# Content-Disposition header default is "{row['_id']['$oid']}/{oid}.{ext}" (no '.json').
# rowpath = row['_id']['$oid']
filepath = os.path.join(row['OBS_NAME'], f"{oid}.{ext}")
url = f"{self._BASE_URL}{inst}_{row['data_type']}.files/{oid}{binfile}"
downloader.enqueue_file(url, filename=str(path).format(file=filepath,
**row.response_block_map))
scope, name = row['links'].split(':')
filepath = os.path.join(row['OBS_NAME'], name)
replicas = self.rucio.list_replicas([{'scope': scope, 'name': name}])
for replica in replicas:
for rse in replica['rses'].values():
for url in rse:
downloader.enqueue_file(url.replace('davs', 'https'),
filename=str(path).format(file=filepath,
**row.response_block_map),
headers=self.rucio.headers)
@classmethod
def _can_handle_query(cls, *query, hasinstr=False):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment