Skip to content

Commit 522873e

Browse files
tomkralidistomkralidis
committed
implement CQL to OGC filter transforms
1 parent c27c247 commit 522873e

7 files changed

+213
-11
lines changed

‎pycsw/cql.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# -*- coding: utf-8 -*-
2+
# =================================================================
3+
#
4+
# Authors: Tom Kralidis <[email protected]>
5+
#
6+
# Copyright (c) 2016 Tom Kralidis
7+
#
8+
# Permission is hereby granted, free of charge, to any person
9+
# obtaining a copy of this software and associated documentation
10+
# files (the "Software"), to deal in the Software without
11+
# restriction, including without limitation the rights to use,
12+
# copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
# copies of the Software, and to permit persons to whom the
14+
# Software is furnished to do so, subject to the following
15+
# conditions:
16+
#
17+
# The above copyright notice and this permission notice shall be
18+
# included in all copies or substantial portions of the Software.
19+
#
20+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
22+
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24+
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27+
# OTHER DEALINGS IN THE SOFTWARE.
28+
#
29+
# =================================================================
30+
31+
import logging
32+
33+
from lxml import etree
34+
from pycsw import util
35+
from pycsw.fes import MODEL as fes1_model
36+
37+
LOGGER = logging.getLogger(__name__)
38+
39+
40+
def cql2fes1(cql, namespaces):
41+
"""transforms Common Query Language (CQL) query into OGC fes1 syntax"""
42+
43+
filters = []
44+
tmp_list = []
45+
logical_op = None
46+
47+
LOGGER.debug('CQL: %s', cql)
48+
49+
if ' or ' in cql:
50+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
51+
tmp_list = cql.split(' or ')
52+
elif ' OR ' in cql:
53+
logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces))
54+
tmp_list = cql.split(' OR ')
55+
elif ' and ' in cql:
56+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
57+
tmp_list = cql.split(' and ')
58+
elif ' AND ' in cql:
59+
logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces))
60+
tmp_list = cql.split(' AND ')
61+
62+
if tmp_list:
63+
LOGGER.debug('Logical operator found (AND/OR)')
64+
else:
65+
tmp_list.append(cql)
66+
67+
for t in tmp_list:
68+
filters.append(_parse_condition(t))
69+
70+
root = etree.Element(util.nspath_eval('ogc:Filter', namespaces))
71+
72+
if logical_op is not None:
73+
root.append(logical_op)
74+
75+
for flt in filters:
76+
condition = etree.Element(util.nspath_eval(flt[0], namespaces))
77+
78+
etree.SubElement(
79+
condition,
80+
util.nspath_eval('ogc:PropertyName', namespaces)).text = flt[1]
81+
82+
etree.SubElement(
83+
condition,
84+
util.nspath_eval('ogc:Literal', namespaces)).text = flt[2]
85+
86+
if logical_op is not None:
87+
logical_op.append(condition)
88+
else:
89+
root.append(condition)
90+
91+
LOGGER.debug('Resulting OGC Filter: %s',
92+
etree.tostring(root, pretty_print=1))
93+
94+
return root
95+
96+
97+
def _parse_condition(condition):
98+
"""parses a single condition"""
99+
100+
LOGGER.debug('condition: %s', condition)
101+
102+
property_name, operator, literal = condition.split()
103+
104+
literal = literal.replace('"', '').replace('\'', '')
105+
106+
for k, v in fes1_model['ComparisonOperators'].items():
107+
if v['opvalue'] == operator:
108+
fes1_predicate = k
109+
110+
LOGGER.debug('parsed condition: %s %s %s', property_name, fes1_predicate,
111+
literal)
112+
113+
return (fes1_predicate, property_name, literal)
114+

‎pycsw/server.py‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from pycsw.plugins.profiles import profile as pprofile
4141
import pycsw.plugins.outputschemas
4242
from pycsw import config, fes, log, metadata, util, sru, oaipmh, opensearch
43+
from pycsw.cql import cql2fes1
4344
import logging
4445

4546
LOGGER = logging.getLogger(__name__)
@@ -1281,12 +1282,20 @@ def getrecords(self):
12811282
% self.kvp['constraintlanguage'])
12821283
if self.kvp['constraintlanguage'] == 'CQL_TEXT':
12831284
tmp = self.kvp['constraint']
1284-
self.kvp['constraint'] = {}
1285-
self.kvp['constraint']['type'] = 'cql'
1286-
self.kvp['constraint']['where'] = \
1287-
self._cql_update_queryables_mappings(tmp,
1288-
self.repository.queryables['_all'])
1289-
self.kvp['constraint']['values'] = {}
1285+
try:
1286+
LOGGER.debug('Transforming CQL into fes1')
1287+
LOGGER.debug('CQL: %s', tmp)
1288+
self.kvp['constraint'] = {}
1289+
self.kvp['constraint']['type'] = 'filter'
1290+
cql = cql2fes1(tmp, self.context.namespaces)
1291+
self.kvp['constraint']['where'], self.kvp['constraint']['values'] = fes.parse(cql,
1292+
self.repository.queryables['_all'], self.repository.dbtype,
1293+
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
1294+
except Exception as err:
1295+
LOGGER.error('Invalid CQL query %s', tmp)
1296+
LOGGER.error('Error message: %s', err, exc_info=True)
1297+
return self.exceptionreport('InvalidParameterValue',
1298+
'constraint', 'Invalid Filter syntax')
12901299
elif self.kvp['constraintlanguage'] == 'FILTER':
12911300
# validate filter XML
12921301
try:
@@ -1364,8 +1373,10 @@ def getrecords(self):
13641373
maxrecords=self.kvp['maxrecords'],
13651374
startposition=int(self.kvp['startposition'])-1)
13661375
except Exception as err:
1376+
LOGGER.debug('Invalid query syntax. Query: %s', self.kvp['constraint'])
1377+
LOGGER.debug('Invalid query syntax. Result: %s', err)
13671378
return self.exceptionreport('InvalidParameterValue', 'constraint',
1368-
'Invalid query: %s' % err)
1379+
'Invalid query syntax')
13691380

13701381
dsresults = []
13711382

@@ -2479,13 +2490,21 @@ def _parse_constraint(self, element):
24792490
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
24802491
except Exception as err:
24812492
return 'Invalid Filter request: %s' % err
2493+
24822494
tmp = element.find(util.nspath_eval('csw:CqlText', self.context.namespaces))
24832495
if tmp is not None:
24842496
LOGGER.debug('CQL specified: %s.' % tmp.text)
2485-
query['type'] = 'cql'
2486-
query['where'] = self._cql_update_queryables_mappings(tmp.text,
2487-
self.repository.queryables['_all'])
2488-
query['values'] = {}
2497+
try:
2498+
LOGGER.debug('Transforming CQL into OGC Filter')
2499+
query['type'] = 'filter'
2500+
cql = cql2fes1(tmp.text, self.context.namespaces)
2501+
query['where'], query['values'] = fes.parse(cql,
2502+
self.repository.queryables['_all'], self.repository.dbtype,
2503+
self.context.namespaces, self.orm, self.language['text'], self.repository.fts)
2504+
except Exception as err:
2505+
LOGGER.error('Invalid CQL request: %s', tmp.text)
2506+
LOGGER.error('Error message: %s', err, exc_info=True)
2507+
return 'Invalid CQL request'
24892508
return query
24902509

24912510
def _test_manager(self):

‎tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_get_GetRecords-filter-cql-title.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="2" numberOfRecordsReturned="2" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="full">
6+
<csw:Record>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
9+
<dc:format>image/svg+xml</dc:format>
10+
<dc:title>Lorem ipsum</dc:title>
11+
<dct:spatial>GR-22</dct:spatial>
12+
<dc:subject>Tourism--Greece</dc:subject>
13+
<dct:abstract>Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu.</dct:abstract>
14+
</csw:Record>
15+
<csw:Record>
16+
<dc:identifier>urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2</dc:identifier>
17+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
18+
<dc:title>Lorem ipsum dolor sit amet</dc:title>
19+
<dc:format>image/jpeg</dc:format>
20+
<dct:spatial>IT-FI</dct:spatial>
21+
</csw:Record>
22+
</csw:SearchResults>
23+
</csw:GetRecordsResponse>

‎tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<!-- PYCSW_VERSION -->
3+
<csw:GetRecordsResponse xmlns:soapenv="http://www.w3.org/2003/05/soap-envelope" xmlns:gml="http://www.opengis.net/gml" xmlns:dif="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ows="http://www.opengis.net/ows" xmlns:fgdc="http://www.opengis.net/cat/csw/csdgm" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:os="http://a9.com/-/spec/opensearch/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ogc="http://www.opengis.net/ogc" version="2.0.2" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
4+
<csw:SearchStatus timestamp="PYCSW_TIMESTAMP"/>
5+
<csw:SearchResults nextRecord="0" numberOfRecordsMatched="1" numberOfRecordsReturned="1" recordSchema="http://www.opengis.net/cat/csw/2.0.2" elementSet="brief">
6+
<csw:BriefRecord>
7+
<dc:identifier>urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f</dc:identifier>
8+
<dc:title>Lorem ipsum</dc:title>
9+
<dc:type>http://purl.org/dc/dcmitype/Image</dc:type>
10+
</csw:BriefRecord>
11+
</csw:SearchResults>
12+
</csw:GetRecordsResponse>

‎tests/suites/default/get/requests.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ GetRecords-filter,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=C
99
Exception-GetRepositoryItem-service-invalid1,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00&version=2.0.2&request=GetRepositoryItem&id=123
1010
Exception-GetRepositoryItem-service-invalid2,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00'&version=2.0.2&request=GetRepositoryItem&id=123
1111
Exception-GetRepositoryItem-version-invalid,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2'&request=GetRepositoryItem&id=123
12+
GetRecords-filter-cql-title,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27
13+
GetRecords-filter-cql-title-or-abstract,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27%20or%20dct%3Aabstract%20like%20%27%25pharetra%25%27

‎tests/suites/default/post/GetRecords-cql-title-and-abstract.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
2+
<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:ogc="http://www.opengis.net/ogc" service="CSW" version="2.0.2" resultType="results" startPosition="1" maxRecords="5" outputFormat="application/xml" outputSchema="http://www.opengis.net/cat/csw/2.0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
3+
<csw:Query typeNames="csw:Record">
4+
<csw:ElementSetName>brief</csw:ElementSetName>
5+
<csw:Constraint version="1.1.0">
6+
<csw:CqlText>dc:title like '%ips%' and dct:abstract like '%pharetra%'</csw:CqlText>
7+
</csw:Constraint>
8+
</csw:Query>
9+
</csw:GetRecords>

0 commit comments

Comments
 (0)