Skip to content

Commit ddb5ef8

Browse files
KamiKami
authored
Merge pull request #4831 from StackStorm/operator_improvements
Update rule comparison operators to work with mixed operator types (bytes and strings)
2 parents f7b9789 + 96b142a commit ddb5ef8

File tree

4 files changed

+150
-3
lines changed

4 files changed

+150
-3
lines changed

‎CHANGELOG.rst‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ Fixed
8888
* Fixed ``core.sendmail`` base64 encoding of longer subject lines (bug fix) #4795
8989

9090
Contributed by @stevemuskiewicz and @guzzijones
91+
* Update all the various rule criteria comparison operators which also work with strings (equals,
92+
icontains, nequals, etc.) to work correctly on Python 3 deployments if one of the operators is
93+
of a type bytes and the other is of a type unicode / string. (bug fix) #4831
9194

9295
3.1.0 - June 27, 2019
9396
---------------------
@@ -131,7 +134,7 @@ Fixed
131134
value for SSH port is specified in the configured SSH config file
132135
(``ssh_runner.ssh_config_file_path``). (bug fix) #4660 #4661
133136
* Update pack install action so it works correctly when ``python_versions`` ``pack.yaml`` metadata
134-
attribute is used in combination with ``--python3`` pack install flag. (bug fix) #4654 #4662
137+
attribute is used in combination with ``--use-python3`` pack install flag. (bug fix) #4654 #4662
135138
* Add ``source_channel`` back to the context used by Mistral workflows for executions which are
136139
triggered via ChatOps (using action alias).
137140

@@ -141,7 +144,7 @@ Fixed
141144
server time where st2api is running was not set to UTC. (bug fix) #4668
142145

143146
Contributed by Igor Cherkaev. (@emptywee)
144-
* Fix a bug with some packs which use ``--python3`` flag (running Python 3 actions on installation
147+
* Fix a bug with some packs which use ``--use-python3`` flag (running Python 3 actions on installation
145148
where StackStorm components run under Python 2) which rely on modules from Python 3 standard
146149
library which are also available in Python 2 site-packages (e.g. ``concurrent``) not working
147150
correctly.

‎contrib/hello_st2/pack.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
---
2-
# Pack reference. It can only contain letters, digits and underscores.
2+
# Pack reference. It can only contain lowercase letters, digits and underscores.
3+
# This attribute is only needed if "name" attribute contains special characters.
34
ref: hello_st2
45
# User-friendly pack name. If this attribute contains spaces or any other special characters, then
56
# the "ref" attribute must also be specified (see above).

‎st2common/st2common/operators.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
from __future__ import absolute_import
16+
1617
import re
1718
import six
1819
import fnmatch
@@ -140,64 +141,85 @@ def search(value, criteria_pattern, criteria_condition, check_function):
140141
def equals(value, criteria_pattern):
141142
if criteria_pattern is None:
142143
return False
144+
145+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
143146
return value == criteria_pattern
144147

145148

146149
def nequals(value, criteria_pattern):
150+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
147151
return value != criteria_pattern
148152

149153

150154
def iequals(value, criteria_pattern):
151155
if criteria_pattern is None:
152156
return False
157+
158+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
153159
return value.lower() == criteria_pattern.lower()
154160

155161

156162
def contains(value, criteria_pattern):
157163
if criteria_pattern is None:
158164
return False
165+
166+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
159167
return criteria_pattern in value
160168

161169

162170
def icontains(value, criteria_pattern):
163171
if criteria_pattern is None:
164172
return False
173+
174+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
165175
return criteria_pattern.lower() in value.lower()
166176

167177

168178
def ncontains(value, criteria_pattern):
169179
if criteria_pattern is None:
170180
return False
181+
182+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
171183
return criteria_pattern not in value
172184

173185

174186
def incontains(value, criteria_pattern):
175187
if criteria_pattern is None:
176188
return False
189+
190+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
177191
return criteria_pattern.lower() not in value.lower()
178192

179193

180194
def startswith(value, criteria_pattern):
181195
if criteria_pattern is None:
182196
return False
197+
198+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
183199
return value.startswith(criteria_pattern)
184200

185201

186202
def istartswith(value, criteria_pattern):
187203
if criteria_pattern is None:
188204
return False
205+
206+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
189207
return value.lower().startswith(criteria_pattern.lower())
190208

191209

192210
def endswith(value, criteria_pattern):
193211
if criteria_pattern is None:
194212
return False
213+
214+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
195215
return value.endswith(criteria_pattern)
196216

197217

198218
def iendswith(value, criteria_pattern):
199219
if criteria_pattern is None:
200220
return False
221+
222+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
201223
return value.lower().endswith(criteria_pattern.lower())
202224

203225

@@ -217,13 +239,16 @@ def match_wildcard(value, criteria_pattern):
217239
if criteria_pattern is None:
218240
return False
219241

242+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
220243
return fnmatch.fnmatch(value, criteria_pattern)
221244

222245

223246
def match_regex(value, criteria_pattern):
224247
# match_regex is deprecated, please use 'regex' and 'iregex'
225248
if criteria_pattern is None:
226249
return False
250+
251+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
227252
regex = re.compile(criteria_pattern, re.DOTALL)
228253
# check for a match and not for details of the match.
229254
return regex.match(value) is not None
@@ -232,6 +257,8 @@ def match_regex(value, criteria_pattern):
232257
def regex(value, criteria_pattern):
233258
if criteria_pattern is None:
234259
return False
260+
261+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
235262
regex = re.compile(criteria_pattern)
236263
# check for a match and not for details of the match.
237264
return regex.search(value) is not None
@@ -240,6 +267,8 @@ def regex(value, criteria_pattern):
240267
def iregex(value, criteria_pattern):
241268
if criteria_pattern is None:
242269
return False
270+
271+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
243272
regex = re.compile(criteria_pattern, re.IGNORECASE)
244273
# check for a match and not for details of the match.
245274
return regex.search(value) is not None
@@ -288,15 +317,40 @@ def nexists(value, criteria_pattern):
288317
def inside(value, criteria_pattern):
289318
if criteria_pattern is None:
290319
return False
320+
321+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
291322
return value in criteria_pattern
292323

293324

294325
def ninside(value, criteria_pattern):
295326
if criteria_pattern is None:
296327
return False
328+
329+
value, criteria_pattern = ensure_operators_are_strings(value, criteria_pattern)
297330
return value not in criteria_pattern
298331

299332

333+
def ensure_operators_are_strings(value, criteria_pattern):
334+
"""
335+
This function ensures that both value and criteria_pattern arguments are unicode (string)
336+
values if the input value type is bytes.
337+
338+
If a value is of types bytes and not a unicode, it's converted to unicode. This way we
339+
ensure all the operators which expect string / unicode values work, even if one of the
340+
values is bytes (this can happen when input is not controlled by the end user - e.g. trigger
341+
payload under Python 3 deployments).
342+
343+
:return: tuple(value, criteria_pattern)
344+
"""
345+
if isinstance(value, bytes):
346+
value = value.decode('utf-8')
347+
348+
if isinstance(criteria_pattern, bytes):
349+
criteria_pattern = criteria_pattern.decode('utf-8')
350+
351+
return value, criteria_pattern
352+
353+
300354
# operator match strings
301355
MATCH_WILDCARD = 'matchwildcard'
302356
MATCH_REGEX = 'matchregex'

‎st2common/tests/unit/test_operators.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,13 @@ def test_matchwildcard(self):
499499
self.assertTrue(op('bar', 'b*r'), 'Failed matchwildcard.')
500500
self.assertTrue(op('bar', 'b?r'), 'Failed matchwildcard.')
501501

502+
# Mixing bytes and strings / unicode should still work
503+
self.assertTrue(op(b'bar', 'b?r'), 'Failed matchwildcard.')
504+
self.assertTrue(op('bar', b'b?r'), 'Failed matchwildcard.')
505+
self.assertTrue(op(b'bar', b'b?r'), 'Failed matchwildcard.')
506+
self.assertTrue(op(u'bar', b'b?r'), 'Failed matchwildcard.')
507+
self.assertTrue(op(u'bar', u'b?r'), 'Failed matchwildcard.')
508+
502509
self.assertFalse(op('1', None), 'Passed matchwildcard with None as criteria_pattern.')
503510

504511
def test_matchregex(self):
@@ -517,13 +524,27 @@ def test_matchregex(self):
517524
string = 'foo\r\nponies\nbar\nfooooo'
518525
self.assertTrue(op(string, '.*ponies.*'), 'Failed matchregex.')
519526

527+
# Mixing bytes and strings / unicode should still work
528+
self.assertTrue(op(b'foo ponies bar', '.*ponies.*'), 'Failed matchregex.')
529+
self.assertTrue(op('foo ponies bar', b'.*ponies.*'), 'Failed matchregex.')
530+
self.assertTrue(op(b'foo ponies bar', b'.*ponies.*'), 'Failed matchregex.')
531+
self.assertTrue(op(b'foo ponies bar', u'.*ponies.*'), 'Failed matchregex.')
532+
self.assertTrue(op(u'foo ponies bar', u'.*ponies.*'), 'Failed matchregex.')
533+
520534
def test_iregex(self):
521535
op = operators.get_operator('iregex')
522536
self.assertTrue(op('V1', 'v1$'), 'Failed iregex.')
523537

524538
string = 'fooPONIESbarfooooo'
525539
self.assertTrue(op(string, 'ponies'), 'Failed iregex.')
526540

541+
# Mixing bytes and strings / unicode should still work
542+
self.assertTrue(op(b'fooPONIESbarfooooo', 'ponies'), 'Failed iregex.')
543+
self.assertTrue(op('fooPONIESbarfooooo', b'ponies'), 'Failed iregex.')
544+
self.assertTrue(op(b'fooPONIESbarfooooo', b'ponies'), 'Failed iregex.')
545+
self.assertTrue(op(b'fooPONIESbarfooooo', u'ponies'), 'Failed iregex.')
546+
self.assertTrue(op(u'fooPONIESbarfooooo', u'ponies'), 'Failed iregex.')
547+
527548
def test_iregex_fail(self):
528549
op = operators.get_operator('iregex')
529550
self.assertFalse(op('V1_foo', 'v1$'), 'Passed iregex.')
@@ -543,6 +564,13 @@ def test_regex(self):
543564
string = 'apple unicorns oranges'
544565
self.assertTrue(op(string, '(ponies|unicorns)'), 'Failed regex.')
545566

567+
# Mixing bytes and strings / unicode should still work
568+
self.assertTrue(op(b'apples unicorns oranges', '(ponies|unicorns)'), 'Failed regex.')
569+
self.assertTrue(op('apples unicorns oranges', b'(ponies|unicorns)'), 'Failed regex.')
570+
self.assertTrue(op(b'apples unicorns oranges', b'(ponies|unicorns)'), 'Failed regex.')
571+
self.assertTrue(op(b'apples unicorns oranges', u'(ponies|unicorns)'), 'Failed regex.')
572+
self.assertTrue(op(u'apples unicorns oranges', u'(ponies|unicorns)'), 'Failed regex.')
573+
546574
string = 'apple unicorns oranges'
547575
self.assertFalse(op(string, '(pikachu|snorlax|charmander)'), 'Passed regex.')
548576

@@ -575,6 +603,13 @@ def test_equals_string(self):
575603
self.assertTrue(op('1', '1'), 'Failed equals.')
576604
self.assertTrue(op('', ''), 'Failed equals.')
577605

606+
# Mixing bytes and strings / unicode should still work
607+
self.assertTrue(op(b'1', '1'), 'Failed equals.')
608+
self.assertTrue(op('1', b'1'), 'Failed equals.')
609+
self.assertTrue(op(b'1', b'1'), 'Failed equals.')
610+
self.assertTrue(op(b'1', u'1'), 'Failed equals.')
611+
self.assertTrue(op(u'1', u'1'), 'Failed equals.')
612+
578613
def test_equals_fail(self):
579614
op = operators.get_operator('equals')
580615
self.assertFalse(op('1', '2'), 'Passed equals.')
@@ -597,6 +632,13 @@ def test_iequals(self):
597632
self.assertTrue(op('ABC', 'abc'), 'Failed iequals.')
598633
self.assertTrue(op('AbC', 'aBc'), 'Failed iequals.')
599634

635+
# Mixing bytes and strings / unicode should still work
636+
self.assertTrue(op(b'AbC', 'aBc'), 'Failed iequals.')
637+
self.assertTrue(op('AbC', b'aBc'), 'Failed iequals.')
638+
self.assertTrue(op(b'AbC', b'aBc'), 'Failed iequals.')
639+
self.assertTrue(op(b'AbC', u'aBc'), 'Failed iequals.')
640+
self.assertTrue(op(u'AbC', u'aBc'), 'Failed iequals.')
641+
600642
def test_iequals_fail(self):
601643
op = operators.get_operator('iequals')
602644
self.assertFalse(op('ABC', 'BCA'), 'Passed iequals.')
@@ -611,6 +653,13 @@ def test_contains(self):
611653
self.assertTrue(op('haystackneedle', 'needle'))
612654
self.assertTrue(op('haystack needle', 'needle'))
613655

656+
# Mixing bytes and strings / unicode should still work
657+
self.assertTrue(op(b'haystack needle', 'needle'))
658+
self.assertTrue(op('haystack needle', b'needle'))
659+
self.assertTrue(op(b'haystack needle', b'needle'))
660+
self.assertTrue(op(b'haystack needle', u'needle'))
661+
self.assertTrue(op(u'haystack needle', b'needle'))
662+
614663
def test_contains_fail(self):
615664
op = operators.get_operator('contains')
616665
self.assertFalse(op('hasystack needl haystack', 'needle'))
@@ -626,6 +675,13 @@ def test_icontains(self):
626675
self.assertTrue(op('haystackNEEDLE', 'needle'))
627676
self.assertTrue(op('haystack needle', 'NEEDLE'))
628677

678+
# Mixing bytes and strings / unicode should still work
679+
self.assertTrue(op(b'haystack needle', 'NEEDLE'))
680+
self.assertTrue(op('haystack needle', b'NEEDLE'))
681+
self.assertTrue(op(b'haystack needle', b'NEEDLE'))
682+
self.assertTrue(op(b'haystack needle', u'NEEDLE'))
683+
self.assertTrue(op(u'haystack needle', b'NEEDLE'))
684+
629685
def test_icontains_fail(self):
630686
op = operators.get_operator('icontains')
631687
self.assertFalse(op('hasystack needl haystack', 'needle'))
@@ -641,6 +697,13 @@ def test_ncontains(self):
641697
self.assertTrue(op('haystackneedle', 'needlex'))
642698
self.assertTrue(op('haystack needle', 'needlex'))
643699

700+
# Mixing bytes and strings / unicode should still work
701+
self.assertTrue(op(b'haystack needle', 'needlex'))
702+
self.assertTrue(op('haystack needle', b'needlex'))
703+
self.assertTrue(op(b'haystack needle', b'needlex'))
704+
self.assertTrue(op(b'haystack needle', u'needlex'))
705+
self.assertTrue(op(u'haystack needle', b'needlex'))
706+
644707
def test_ncontains_fail(self):
645708
op = operators.get_operator('ncontains')
646709
self.assertFalse(op('hasystack needle haystack', 'needle'))
@@ -667,6 +730,13 @@ def test_startswith(self):
667730
self.assertTrue(op('hasystack needle haystack', 'hasystack'))
668731
self.assertTrue(op('a hasystack needle haystack', 'a '))
669732

733+
# Mixing bytes and strings / unicode should still work
734+
self.assertTrue(op(b'haystack needle', 'haystack'))
735+
self.assertTrue(op('haystack needle', b'haystack'))
736+
self.assertTrue(op(b'haystack needle', b'haystack'))
737+
self.assertTrue(op(b'haystack needle', u'haystack'))
738+
self.assertTrue(op(u'haystack needle', b'haystack'))
739+
670740
def test_startswith_fail(self):
671741
op = operators.get_operator('startswith')
672742
self.assertFalse(op('hasystack needle haystack', 'needle'))
@@ -678,6 +748,13 @@ def test_istartswith(self):
678748
self.assertTrue(op('haystack needle haystack', 'HAYstack'))
679749
self.assertTrue(op('HAYSTACK needle haystack', 'haystack'))
680750

751+
# Mixing bytes and strings / unicode should still work
752+
self.assertTrue(op(b'HAYSTACK needle haystack', 'haystack'))
753+
self.assertTrue(op('HAYSTACK needle haystack', b'haystack'))
754+
self.assertTrue(op(b'HAYSTACK needle haystack', b'haystack'))
755+
self.assertTrue(op(b'HAYSTACK needle haystack', u'haystack'))
756+
self.assertTrue(op(u'HAYSTACK needle haystack', b'haystack'))
757+
681758
def test_istartswith_fail(self):
682759
op = operators.get_operator('istartswith')
683760
self.assertFalse(op('hasystack needle haystack', 'NEEDLE'))
@@ -689,6 +766,13 @@ def test_endswith(self):
689766
self.assertTrue(op('hasystack needle haystackend', 'haystackend'))
690767
self.assertTrue(op('a hasystack needle haystack b', 'b'))
691768

769+
# Mixing bytes and strings / unicode should still work
770+
self.assertTrue(op(b'a hasystack needle haystack b', 'b'))
771+
self.assertTrue(op('a hasystack needle haystack b', b'b'))
772+
self.assertTrue(op(b'a hasystack needle haystack b', b'b'))
773+
self.assertTrue(op(b'a hasystack needle haystack b', u'b'))
774+
self.assertTrue(op(u'a hasystack needle haystack b', b'b'))
775+
692776
def test_endswith_fail(self):
693777
op = operators.get_operator('endswith')
694778
self.assertFalse(op('hasystack needle haystackend', 'haystack'))
@@ -776,6 +860,11 @@ def test_inside(self):
776860
self.assertFalse(op('a', 'bcd'), 'Should return False')
777861
self.assertTrue(op('a', 'abc'), 'Should return True')
778862

863+
# Mixing bytes and strings / unicode should still work
864+
self.assertTrue(op(b'a', 'abc'), 'Should return True')
865+
self.assertTrue(op('a', b'abc'), 'Should return True')
866+
self.assertTrue(op(b'a', b'abc'), 'Should return True')
867+
779868
def test_ninside(self):
780869
op = operators.get_operator('ninside')
781870
self.assertFalse(op('a', None), 'Should return False')

0 commit comments

Comments
 (0)