Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrew Leonard
kisfido
Commits
72e3649d
Commit
72e3649d
authored
Oct 18, 2021
by
Derek Homeier
Browse files
Add 'fullrange' option for Range attribute queries
parent
4c8345e1
Pipeline
#2428
failed with stage
in 3 minutes and 30 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
python-user-api/client.py
View file @
72e3649d
...
...
@@ -94,19 +94,24 @@ def _update_val(dictionary, key, value, block='description', regex=False):
return
dictionary
.
update
({
key
:
query
})
def
_update_range
(
dictionary
,
key
,
values
,
block
=
'description'
):
def
_update_range
(
dictionary
,
key
,
values
,
block
=
'description'
,
fullrange
=
False
):
"""
Update dictionary with field_name:{value range} string in format parseable by BSON filter.
"""
if
isinstance
(
values
[
0
],
(
int
,
float
,
complex
)):
query
=
f
"{{'
{
block
}
.
{
key
}
':{{'$gt':{{
{
values
[
0
]:
g
}
}},'$lt':{{
{
values
[
1
]:
g
}
}}}}}}"
# Special case for fields having a _MIN and _MAX form: MIN <= values[1], MAX >= values[0]
strvals
=
[
_str_val
(
values
[
0
]),
_str_val
(
values
[
1
])]
# Special case for fields having a _MIN and _MAX form:
if
key
.
endswith
(
'_'
):
query
=
(
f
"{{'
{
block
}
.
{
key
}
MIN':{{'$le':{{
{
_str_val
(
values
[
1
])
}
}}}}}},"
f
"{{'
{
block
}
.
{
key
}
MAX':{{'$ge':{{
{
_str_val
(
values
[
0
])
}
}}}}}}"
)
if
fullrange
:
# Cover full search range: MIN >= values[:] >= MAX
query
=
(
f
"{{'
{
block
}
.
{
key
}
MIN':{{'$le':{{
{
strvals
[
0
]
}
}}}}}},"
f
"{{'
{
block
}
.
{
key
}
MAX':{{'$ge':{{
{
strvals
[
1
]
}
}}}}}}"
)
else
:
# Default: MIN <= values[1], MAX >= values[0]
query
=
(
f
"{{'
{
block
}
.
{
key
}
MIN':{{'$le':{{
{
strvals
[
1
]
}
}}}}}},"
f
"{{'
{
block
}
.
{
key
}
MAX':{{'$ge':{{
{
strvals
[
0
]
}
}}}}}}"
)
else
:
query
=
(
f
"{{'
{
block
}
.
{
key
}
':{{'$ge':{{
{
_
str
_
val
(
value
s
[
0
]
)
}
}},"
f
"'$le':{{
{
_
str
_
val
(
value
s
[
1
]
)
}
}}}}}}"
)
query
=
(
f
"{{'
{
block
}
.
{
key
}
':{{'$ge':{{
{
strvals
[
0
]
}
}},"
f
"'$le':{{
{
strvals
[
1
]
}
}}}}}}"
)
return
dictionary
.
update
({
key
:
query
})
...
...
@@ -118,7 +123,7 @@ def _update_range(dictionary, key, values, block='description'):
def
_
(
wlk
,
attr
,
params
):
"""Set `description.TYPE_NAME`"""
key
=
_obs_fields
.
get
(
attr
.
type_name
,
attr
.
type_name
.
upper
())
return
_update_val
(
params
,
key
,
attr
.
value
)
return
_update_val
(
params
,
key
,
attr
.
value
,
regex
=
getattr
(
attr
,
'regex'
,
False
)
)
# OBS_COLLECTION: `instrument_observations` also defines the MongoDB collection (base path).
...
...
@@ -127,7 +132,7 @@ def _(wlk, attr, params):
"""Set 'description.INSTRUMENT' and 'description.OBS_COLLECTION'"""
params
.
update
({
'OBS_COLLECTION'
:
f
'
{
attr
.
value
.
lower
()
}
_observations'
})
key
=
attr
.
type_name
.
upper
()
return
_update_val
(
params
,
key
,
attr
.
value
)
return
_update_val
(
params
,
key
,
attr
.
value
,
regex
=
getattr
(
attr
,
'regex'
,
False
)
)
# Time range covering the observation - require at least part of it within 'DATE_BEG', 'DATE_END'
...
...
@@ -135,9 +140,10 @@ def _(wlk, attr, params):
def
_
(
wlk
,
attr
,
params
):
"""Set 'description.DATE_BEG|END' [ISO Time str]"""
if
attr
.
end
==
attr
.
start
:
_update_val
(
params
,
'DATE_'
,
attr
.
start
)
_update_val
(
params
,
'DATE_'
,
attr
.
start
,
regex
=
getattr
(
attr
,
'regex'
,
False
)
)
else
:
_update_range
(
params
,
'DATE_'
,
[
attr
.
start
,
attr
.
end
])
_update_range
(
params
,
'DATE_'
,
[
attr
.
start
,
attr
.
end
],
fullrange
=
getattr
(
attr
,
'fullrange'
,
False
))
params
[
'DATE_'
]
=
params
[
'DATE_'
].
replace
(
'_MIN'
,
'_BEG'
).
replace
(
'_MAX'
,
'_END'
)
return
...
...
@@ -146,7 +152,7 @@ def _(wlk, attr, params):
@
walker
.
add_applier
(
sattrs
.
Date
)
def
_
(
wlk
,
attr
,
params
):
"""Set 'description.DATE_BEG|END' [ISO Time str]"""
_update_val
(
params
,
attr
.
type_name
.
upper
(),
attr
.
time
)
_update_val
(
params
,
attr
.
type_name
.
upper
(),
attr
.
time
,
regex
=
getattr
(
attr
,
'regex'
,
False
)
)
params
[
'DATE_'
]
=
params
[
'DATE_'
].
replace
(
'_MIN'
,
'_BEG'
).
replace
(
'_MAX'
,
'_END'
)
return
...
...
@@ -169,7 +175,8 @@ def _(wlk, attr, params):
if
attrange
[
1
]
==
attrange
[
0
]:
return
_update_val
(
params
,
key
,
attrange
[
0
])
else
:
return
_update_range
(
params
,
key
,
[
min
(
attrange
),
max
(
attrange
)])
attrange
=
[
min
(
attrange
),
max
(
attrange
)]
return
_update_range
(
params
,
key
,
attrange
,
fullrange
=
getattr
(
attr
,
'fullrange'
,
False
))
@
walker
.
add_applier
(
a
.
Provider
,
a
.
ExtentType
,
a
.
Source
)
...
...
python-user-api/tests/test_client.py
View file @
72e3649d
...
...
@@ -24,7 +24,7 @@ def client():
def
test_search
(
client
):
"""Test conversion of Attrs to query string."""
"""Test conversion of
(supported)
Attrs to query string."""
assert
not
client
.
_can_handle_query
(
a
.
Time
(
"2019/01/01"
,
"2021/01/01"
))
with
pytest
.
raises
(
AttributeError
,
match
=
r
"Query not possible: "
...
...
@@ -102,7 +102,7 @@ def test_cant_handle_query(client, query):
a
.
sdc
.
SpatialXel1
(
200
,
3000
),
a
.
sdc
.
SpatialXel2
(
100
,
4000
),
a
.
sdc
.
SpectralXel
(
320
,
4096
),
a
.
sdc
.
TimeXel
(
60
,
86400
)))
def
test_all_queries
(
client
,
query
):
"""Test an example of all supported query attributes."""
"""Test an example of all supported query attributes
with automatic field names
."""
assert
client
.
_can_handle_query
(
a
.
Instrument
(
"GRIS"
),
query
)
if
HAS_DOCKERTEST
:
res
=
client
.
search
(
a
.
Instrument
(
"GRIS"
)
&
query
)
...
...
@@ -112,3 +112,38 @@ def test_all_queries(client, query):
r
"{'.and':.{'description.INSTRUMENT':'gris'},"
rf
"{{'description.*
{
query
.
type_name
.
upper
()
}
"
):
client
.
search
(
a
.
Instrument
(
"GRIS"
)
&
query
)
def
test_full_range
(
client
):
"""
Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`.
"""
t
=
a
.
Time
(
"2019/01/01"
,
"2019/01/09"
)
t
.
fullrange
=
True
query
=
a
.
Instrument
(
"BBI"
)
&
t
assert
client
.
_can_handle_query
(
query
)
if
HAS_DOCKERTEST
:
res
=
client
.
search
(
query
)
else
:
with
pytest
.
raises
(
URLError
,
match
=
r
"Unable to execute search "
r
".http://dockertest:8083/sdc/bbi_observations.filter="
r
"{'.and':.{'description.INSTRUMENT':'bbi'},"
r
"{'description.DATE_BEG':{'.le':{'2019-01-01T00:00:00.000'}}},"
r
"{'description.DATE_END':{'.ge':{'2019-01-09T00:00:00.000'}}}.}"
):
client
.
search
(
query
)
# Test with inverted `min`, `max` inputs.
hplt
=
a
.
sdc
.
HelioProjLat
(
0.2
*
u
.
arcmin
,
-
10
*
u
.
arcsec
)
hplt
.
fullrange
=
True
query
=
a
.
Instrument
(
"LARS"
)
&
hplt
assert
client
.
_can_handle_query
(
query
)
if
HAS_DOCKERTEST
:
res
=
client
.
search
(
query
)
else
:
with
pytest
.
raises
(
URLError
,
match
=
r
"Unable to execute search "
r
".http://dockertest:8083/sdc/lars_observations.filter="
r
"{'.and':.{'description.INSTRUMENT':'lars'},"
r
"{'description.HPLT_TAN_MIN':{'.le':{-10\}}},"
r
"{'description.HPLT_TAN_MAX':{'.ge':{12\}}}"
):
client
.
search
(
query
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment