Skip to content

Commit 69546e1

Browse files
tomkralidistomkralidis
committed
implement CQL to OGC filter transforms
1 parent d1a79ec commit 69546e1

File tree

8 files changed

+244
-24
lines changed

8 files changed

+244
-24
lines changed

‎pycsw/ogc/csw/cql.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# -*- coding: utf-8 -*-
2+
# =================================================================
3+
#
4+
# Authors: Tom Kralidis <[email protected]>
5+
#
6+
# Copyright (c) 2016 Tom Kralidis
7+
#
8+
# Permission is hereby granted, free of charge, to any person
9+
# obtaining a copy of this software and associated documentation
10+
# files (the "Software"), to deal in the Software without
11+
# restriction, including without limitation the rights to use,
12+
# copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
# copies of the Software, and to permit persons to whom the
14+
# Software is furnished to do so, subject to the following
15+
# conditions:
16+
#
17+
# The above copyright notice and this permission notice shall be
18+
# included in all copies or substantial portions of the Software.
19+
#
20+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
22+
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24+
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27+
# OTHER DEALINGS IN THE SOFTWARE.
28+
#
29+
# =================================================================
30+
31+
import logging
32+
33+
from pycsw.core.etree import etree
34+
from pycsw.core import util
35+
from pycsw.ogc.fes.fes1 import MODEL as fes1_model
36+
37+
LOGGER = logging.getLogger(__name__)
38+
39+
40+
def cql2fes1(cql, namespaces):
41+
"""transforms Common Query Language (CQL) query into OGC fes1 syntax"""
42+
43+
filters = []
44+
tmp_list = []
45+
logical_op = None
46+
47+
LOGGER.debug('CQL: %s', cql)
48+
49+
if ' or ' in cql:
50+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
51+
tmp_list = cql.split(' or ')
52+
elif ' OR ' in cql:
53+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
54+
tmp_list = cql.split(' OR ')
55+
elif ' and ' in cql:
56+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
57+
tmp_list = cql.split(' and ')
58+
elif ' AND ' in cql:
59+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
60+
tmp_list = cql.split(' AND ')
61+
62+
if tmp_list:
63+
LOGGER.debug('Logical operator found (AND/OR)')
64+
else:
65+
tmp_list.append(cql)
66+
67+
for t in tmp_list:
68+
filters.append(_parse_condition(t))
69+
70+
root = etree.Element(util.nspath_eval('ogc:Filter', namespaces))
71+
72+
if logical_op is not None:
73+
root.append(logical_op)
74+
75+
for flt in filters:
76+
condition = etree.Element(util.nspath_eval(flt[0], namespaces))
77+
78+
etree.SubElement(
79+
condition,
80+
util.nspath_eval('ogc:PropertyName', namespaces)).text = flt[1]
81+
82+
etree.SubElement(
83+
condition,
84+
util.nspath_eval('ogc:Literal', namespaces)).text = flt[2]
85+
86+
if logical_op is not None:
87+
logical_op.append(condition)
88+
else:
89+
root.append(condition)
90+
91+
LOGGER.debug('Resulting OGC Filter: %s',
92+
etree.tostring(root, pretty_print=1))
93+
94+
return root
95+
96+
97+
def _parse_condition(condition):
98+
"""parses a single condition"""
99+
100+
LOGGER.debug('condition: %s', condition)
101+
102+
property_name, operator, literal = condition.split()
103+
104+
literal = literal.replace('"', '').replace('\'', '')
105+
106+
for k, v in fes1_model['ComparisonOperators'].items():
107+
if v['opvalue'] == operator:
108+
fes1_predicate = k
109+
110+
LOGGER.debug('parsed condition: %s %s %s', property_name, fes1_predicate,
111+
literal)
112+
113+
return (fes1_predicate, property_name, literal)

‎pycsw/ogc/csw/csw2.py‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
from six.moves.configparser import SafeConfigParser
3939
from pycsw.core.etree import etree
4040
from pycsw import oaipmh, opensearch, sru
41+
from pycsw.ogc.csw.cql import cql2fes1
4142
from pycsw.plugins.profiles import profile as pprofile
4243
import pycsw.plugins.outputschemas
4344
from pycsw.core import config, log, metadata, util
@@ -732,12 +733,20 @@ def getrecords(self):
732733
% self.parent.kvp['constraintlanguage'])
733734
if self.parent.kvp['constraintlanguage'] == 'CQL_TEXT':
734735
tmp = self.parent.kvp['constraint']
735-
self.parent.kvp['constraint'] = {}
736-
self.parent.kvp['constraint']['type'] = 'cql'
737-
self.parent.kvp['constraint']['where'] = \
738-
self.parent._cql_update_queryables_mappings(tmp,
739-
self.parent.repository.queryables['_all'])
740-
self.parent.kvp['constraint']['values'] = {}
736+
try:
737+
LOGGER.debug('Transforming CQL into fes1')
738+
LOGGER.debug('CQL: %s', tmp)
739+
self.parent.kvp['constraint'] = {}
740+
self.parent.kvp['constraint']['type'] = 'filter'
741+
cql = cql2fes1(tmp, self.parent.context.namespaces)
742+
self.parent.kvp['constraint']['where'], self.parent.kvp['constraint']['values'] = fes1.parse(cql,
743+
self.parent.repository.queryables['_all'], self.parent.repository.dbtype,
744+
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
745+
except Exception as err:
746+
LOGGER.error('Invalid CQL query %s', tmp)
747+
LOGGER.error('Error message: %s', err, exc_info=True)
748+
return self.exceptionreport('InvalidParameterValue',
749+
'constraint', 'Invalid Filter syntax')
741750
elif self.parent.kvp['constraintlanguage'] == 'FILTER':
742751
# validate filter XML
743752
try:
@@ -815,8 +824,10 @@ def getrecords(self):
815824
maxrecords=self.parent.kvp['maxrecords'],
816825
startposition=int(self.parent.kvp['startposition'])-1)
817826
except Exception as err:
827+
LOGGER.debug('Invalid query syntax. Query: %s', self.parent.kvp['constraint'])
828+
LOGGER.debug('Invalid query syntax. Result: %s', err)
818829
return self.exceptionreport('InvalidParameterValue', 'constraint',
819-
'Invalid query: %s' % err)
830+
'Invalid query syntax')
820831

821832
dsresults = []
822833

@@ -1536,13 +1547,21 @@ def _parse_constraint(self, element):
15361547
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
15371548
except Exception as err:
15381549
return 'Invalid Filter request: %s' % err
1550+
15391551
tmp = element.find(util.nspath_eval('csw:CqlText', self.parent.context.namespaces))
15401552
if tmp is not None:
1541-
LOGGER.debug('CQL specified: %s.' % tmp.text)
1542-
query['type'] = 'cql'
1543-
query['where'] = self.parent._cql_update_queryables_mappings(tmp.text,
1544-
self.parent.repository.queryables['_all'])
1545-
query['values'] = {}
1553+
LOGGER.debug('CQL specified: %s.', tmp.text)
1554+
try:
1555+
LOGGER.debug('Transforming CQL into OGC Filter')
1556+
query['type'] = 'filter'
1557+
cql = cql2fes1(tmp.text, self.parent.context.namespaces)
1558+
query['where'], query['values'] = fes1.parse(cql,
1559+
self.parent.repository.queryables['_all'], self.parent.repository.dbtype,
1560+
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
1561+
except Exception as err:
1562+
LOGGER.error('Invalid CQL request: %s', tmp.text)
1563+
LOGGER.error('Error message: %s', err, exc_info=True)
1564+
return 'Invalid CQL request'
15461565
return query
15471566

15481567
def parse_postdata(self, postdata):

‎pycsw/ogc/csw/csw3.py‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from six import StringIO
3737
from six.moves.configparser import SafeConfigParser
3838
from pycsw.core.etree import etree
39+
from pycsw.ogc.csw.cql import cql2fes1
3940
from pycsw import oaipmh, opensearch, sru
4041
from pycsw.plugins.profiles import profile as pprofile
4142
import pycsw.plugins.outputschemas
@@ -759,12 +760,20 @@ def getrecords(self):
759760
% self.parent.kvp['constraintlanguage'])
760761
if self.parent.kvp['constraintlanguage'] == 'CQL_TEXT':
761762
tmp = self.parent.kvp['constraint']
762-
self.parent.kvp['constraint'] = {}
763-
self.parent.kvp['constraint']['type'] = 'cql'
764-
self.parent.kvp['constraint']['where'] = \
765-
self.parent._cql_update_queryables_mappings(tmp,
766-
self.parent.repository.queryables['_all'])
767-
self.parent.kvp['constraint']['values'] = {}
763+
try:
764+
LOGGER.debug('Transforming CQL into fes1')
765+
LOGGER.debug('CQL: %s', tmp)
766+
self.parent.kvp['constraint'] = {}
767+
self.parent.kvp['constraint']['type'] = 'filter'
768+
cql = cql2fes1(tmp, self.parent.context.namespaces)
769+
self.parent.kvp['constraint']['where'], self.parent.kvp['constraint']['values'] = fes1.parse(cql,
770+
self.parent.repository.queryables['_all'], self.parent.repository.dbtype,
771+
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
772+
except Exception as err:
773+
LOGGER.error('Invalid CQL query %s', tmp)
774+
LOGGER.error('Error message: %s', err, exc_info=True)
775+
return self.exceptionreport('InvalidParameterValue',
776+
'constraint', 'Invalid Filter syntax')
768777
elif self.parent.kvp['constraintlanguage'] == 'FILTER':
769778
# validate filter XML
770779
try:
@@ -851,8 +860,10 @@ def getrecords(self):
851860
maxrecords=self.parent.kvp['maxrecords'],
852861
startposition=int(self.parent.kvp['startposition'])-1)
853862
except Exception as err:
863+
LOGGER.debug('Invalid query syntax. Query: %s', self.parent.kvp['constraint'])
864+
LOGGER.debug('Invalid query syntax. Result: %s', err)
854865
return self.exceptionreport('InvalidParameterValue', 'constraint',
855-
'Invalid query: %s' % err)
866+
'Invalid query syntax')
856867

857868
if int(matched) == 0:
858869
returned = nextrecord = '0'
@@ -1616,13 +1627,21 @@ def _parse_constraint(self, element):
16161627
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
16171628
except Exception as err:
16181629
return 'Invalid Filter request: %s' % err
1630+
16191631
tmp = element.find(util.nspath_eval('csw30:CqlText', self.parent.context.namespaces))
16201632
if tmp is not None:
1621-
LOGGER.debug('CQL specified: %s.' % tmp.text)
1622-
query['type'] = 'cql'
1623-
query['where'] = self.parent._cql_update_queryables_mappings(tmp.text,
1624-
self.parent.repository.queryables['_all'])
1625-
query['values'] = {}
1633+
LOGGER.debug('CQL specified: %s.', tmp.text)
1634+
try:
1635+
LOGGER.debug('Transforming CQL into OGC Filter')
1636+
query['type'] = 'filter'
1637+
cql = cql2fes1(tmp.text, self.parent.context.namespaces)
1638+
query['where'], query['values'] = fes1.parse(cql,
1639+
self.parent.repository.queryables['_all'], self.parent.repository.dbtype,
1640+
self.parent.context.namespaces, self.parent.orm, self.parent.language['text'], self.parent.repository.fts)
1641+
except Exception as err:
1642+
LOGGER.error('Invalid CQL request: %s', tmp.text)
1643+
LOGGER.error('Error message: %s', err, exc_info=True)
1644+
return 'Invalid CQL request'
16261645
return query
16271646

16281647
def parse_postdata(self, postdata):

‎tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:gml="http://www.opengis.net/gml" xmlns:ows="http://www.opengis.net/ows" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_get_GetRecords-filter-cql-title.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:gml="http://www.opengis.net/gml" xmlns:ows="http://www.opengis.net/ows" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:gml="http://www.opengis.net/gml" xmlns:ows="http://www.opengis.net/ows" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="1" numberOfRecordsReturned="1" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="brief">
6+
<csw:BriefRecord>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:title>Lorem ipsum</dc:title>
9+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
10+
</csw:BriefRecord>
11+
</csw:SearchResults>
12+
</csw:GetRecordsResponse>

‎tests/suites/default/get/requests.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ GetRecords-sortby-desc,PYCSW_SERVER?config=tests/suites/default/default.cfg&serv
66
GetRecords-sortby-invalid-propertyname,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&sortby=dc:titlei:A
77
GetRecords-sortby-invalid-order,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&sortby=dc:title:FOO
88
GetRecords-filter,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=FILTER&constraint=%3Cogc%3AFilter%20xmlns%3Aogc%3D%22http%3A%2F%2Fwww.opengis.net%2Fogc%22%3E%3Cogc%3APropertyIsEqualTo%3E%3Cogc%3APropertyName%3Edc%3Atitle%3C%2Fogc%3APropertyName%3E%3Cogc%3ALiteral%3ELorem%20ipsum%3C%2Fogc%3ALiteral%3E%3C%2Fogc%3APropertyIsEqualTo%3E%3C%2Fogc%3AFilter%3E
9+
GetRecords-filter-cql-title,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27
10+
GetRecords-filter-cql-title-or-abstract,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27%20or%20dct%3Aabstract%20like%20%27%25pharetra%25%27
911
GetRecords-empty-maxrecords,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&maxrecords=
1012
GetRepositoryItem,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRepositoryItem&id=urn:uuid:94bc9c83-97f6-4b40-9eb8-a8e8787a5c63
1113
Exception-GetRepositoryItem-notfound,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRepositoryItem&id=NOTFOUND

‎tests/suites/default/post/GetRecords-cql-title-and-abstract.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
2+
<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:ogc="http://www.opengis.net/ogc" service="CSW" version="2.0.2" resultType="results" startPosition="1" maxRecords="5" outputFormat="application/xml" outputSchema="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
3+
<csw:Query typeNames="csw:Record">
4+
<csw:ElementSetName>brief</csw:ElementSetName>
5+
<csw:Constraint version="1.1.0">
6+
<csw:CqlText>dc:title like '%ips%' and dct:abstract like '%pharetra%'</csw:CqlText>
7+
</csw:Constraint>
8+
</csw:Query>
9+
</csw:GetRecords>

0 commit comments

Comments
 (0)