1 import copy
2 import re
3 import difflib
4 import json
5 import pytest
6 from collections import OrderedDict
7 from warnings import warn
8 from tlib.base.TestHelper import sort_list, sort_dict
9 import jsonpath_rw
10 from tlib.base.ExceptionHelper import BadJsonFormatError
11 from datadiff import diff
16 """
17 Functions for modifying and validating JSON data
18 """
19
20 @staticmethod
22 """
23 Compares two JSON values\n
24 If parameter validate_order is True, position of keys is taken into account during validation\n
25 If json1 or json2 are strings, it must be a valid JSON string
26
27 @param json1: String or dictionary to compare
28 @type json1: str, dict
29 @param json2: String or dictionary to compare
30 @type json2: str, dict
31 @param ignore_order: If true, order of keys is not validated
32 @type ignore_order: bool
33 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1'
34 @type exclude_keys: list
35 @raise AssertionError: JSON values doesn't match
36 """
37
38 j1 = copy.deepcopy(json1)
39 j2 = copy.deepcopy(json2)
40
41 if ignore_order:
42 JsonHelper._validate_json_ignoring_order(j1, j2, exclude_keys)
43 else:
44 JsonHelper._validate_json_with_order(j1, j2, exclude_keys)
45
46 @staticmethod
48 """
49 Compares two JSON values, ignoring position of keys\n
50 If json1 or json2 are strings, it must be a valid JSON string
51 @param json1: String or dictionary to compare
52 @type json1: str, dict
53 @param json2: String or dictionary to compare
54 @type json2: str, dict
55 @raise AssertionError: JSON values doesn't match
56 """
57 items = [json1, json2]
58
59 for i, item in enumerate(items):
60
61 if isinstance(item, basestring):
62 try:
63 item = json.loads(item)
64 except ValueError:
65 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
66
67
68 if type(item) is dict:
69
70 item = sort_dict(item)
71 elif type(item) is list:
72
73 item = sort_list(item)
74 elif type(item) is tuple:
75 pass
76 else:
77 raise RuntimeError("Parameter doesn't represent a valid JSON object: %s" % item)
78
79 if type(item) in (dict, list):
80
81 JsonHelper.remove_keys(item, exclude_keys)
82
83 items[i] = item
84
85 JsonHelper.assert_equal(items[0], items[1])
86
87 @staticmethod
89 """
90 Compares two JSON values, taking into account key order
91 If json1 or json2 are strings, it must be a valid JSON string
92 @param json1: String to compare
93 @type json1: str
94 @param json2: String to compare
95 @type json2: str
96 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1'
97 @type exclude_keys: list
98 @raise AssertionError: JSON values doesn't match
99 """
100
101 def object_pairs_hook(values):
102 """
103 Method that stores objects in an OrderedDict so comparison takes into account key order
104 """
105 return OrderedDict(values)
106
107 items = [json1, json2]
108 for i, item in enumerate(items):
109 if not isinstance(item, basestring):
110 raise RuntimeError("Only strings are allowed when validating key order")
111
112
113
114 try:
115 item = json.loads(item, object_pairs_hook=object_pairs_hook)
116 except ValueError:
117 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
118
119
120 JsonHelper.remove_keys(item, exclude_keys)
121
122 items[i] = item
123
124
125 if items[0] != items[1]:
126 json1 = json.dumps(items[0], indent=3).splitlines(True)
127 json2 = json.dumps(items[1], indent=3).splitlines(True)
128 diff = difflib.unified_diff(json1, json2)
129 diff_txt = "".join(diff)
130
131 raise AssertionError(r"JSON values didn't match\nValue1:\n%s\n\nValue2:\n%s\n\nDiff:\n%s""" %
132 ("".join(json1), "".join(json2), diff_txt))
133
134 @staticmethod
136 """
137 Takes two dictionaries, compares and displays any distinct keys in both, matching keys but different values, matching key/values
138 @param dictA: dictionary or tuple (for e.g. expected dictionary)
139 @type dictA: dictionary
140 @param dictB: dictionary or tuple (for e.g. dictionary captured at run time)
141 @type dictA: dictionary
142 e.g. Tuple: dictA = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['en']}, {'drink': ['water'], 'weather': ['rainy']}, {'device': ['iPad']}
143 dictB = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['fr'], 'color':['blue']}, {'drink': ['alcohol'], 'weather': ['rainy']}, {'device': ['iPad']}
144 Output: (prints only the non matching dictionaries with the differences)
145 > assert False, '\n'.join(nonmatching_dict)
146 E AssertionError:
147 E
148 E Dictionary_1
149 E distinct_tags_in_dict_1A: None
150 E distinct_tags_in_dict_1B: {color:['blue']}
151 E nonmatching_tags_in_two_dicts:{ln:['en']} & {ln:['fr']} respectively
152 E matching_tags_in_two_dicts:{city:['montreal']}
153 E
154 E
155 E Dictionary_2
156 E distinct_tags_in_dict_2A: None
157 E distinct_tags_in_dict_2B: None
158 E nonmatching_tags_in_two_dicts:{drink:['water']} & {drink:['alcohol']} respectively
159 E matching_tags_in_two_dicts:{weather:['rainy']}
160 """
161
162 nonmatching_dict = []
163 matching_dict = []
164
165 distinct_tags_in_dictA = []
166 distinct_tags_in_dictB = []
167 matching_tags_in_two_dicts = []
168 nonmatching_tags_in_two_dicts = []
169
170 myflag = []
171 i = 0
172 tot_dict = 0
173
174
175 if type(dictA) is dict:
176 tot_dict = 1
177 dictA = eval('[%s]' % dictA)
178 dictB = eval('[%s]' % dictB)
179 elif type(dictA) is tuple or list:
180 tot_dict = len(dictA)
181
182
183 for i in range(tot_dict):
184 differences = diff(dictA[i], dictB[i])
185 for dif in differences.diffs:
186 if dif[0] == 'context_end_container':
187 break
188 else:
189
190 key = dif[1][0][0]
191 expected_val = dictA[i].get(key)
192 captured_val = dictB[i].get(key)
193
194
195 if dif[0] == 'insert':
196
197 distinct_tags_in_dictB.append("{%s:%s}" %(key, captured_val))
198
199
200 elif dif[0] == 'delete':
201 myflag.append("False")
202 distinct_tags_in_dictA.append("{%s:%s}" %(key, expected_val))
203
204 elif dif[0] == 'equal':
205
206 if expected_val != captured_val:
207 myflag.append("False")
208 nonmatching_tags_in_two_dicts.append("{%s:%s} & {%s:%s} respectively" %(key, expected_val,key, captured_val))
209
210
211 else:
212 myflag.append("True")
213 matching_tags_in_two_dicts.append("{%s:%s}" % (key, expected_val))
214
215
216 elif expected_val != captured_val:
217 myflag.append("False")
218 nonmatching_tags_in_two_dicts.append("{%s:%s}/{%s:%s}" %(key, expected_val, key, captured_val))
219
220 if (distinct_tags_in_dictA and distinct_tags_in_dictB) or (nonmatching_tags_in_two_dicts) != [] and ("False" in myflag):
221 if distinct_tags_in_dictA ==[]: distinct_tags_in_dictA.append("None")
222 if distinct_tags_in_dictB ==[]: distinct_tags_in_dictB.append("None")
223 if nonmatching_tags_in_two_dicts ==[]: nonmatching_tags_in_two_dicts.append("None")
224 if matching_tags_in_two_dicts ==[]: matching_tags_in_two_dicts.append("None")
225 nonmatching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "distinct_tags_in_dict_%dA: " % int(i+1) + ','.join(distinct_tags_in_dictA) + '\n' + "distinct_tags_in_dict_%dB: " % int(i+1) + ','.join(distinct_tags_in_dictB) + '\n' + "nonmatching_tags_in_two_dicts:" + ','.join(nonmatching_tags_in_two_dicts) + '\n' + "matching_tags_in_two_dicts:" + ','.join(matching_tags_in_two_dicts))
226 elif "True" in myflag:
227 matching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "matching_tags_in_two_dicts: " + ','.join(matching_tags_in_two_dicts))
228
229
230 distinct_tags_in_dictA = []
231 distinct_tags_in_dictB = []
232 matching_tags_in_two_dicts = []
233 nonmatching_tags_in_two_dicts = []
234
235 if "False" in myflag:
236 assert False, '\n'.join(nonmatching_dict)
237
238
239 @staticmethod
241 """
242 Takes a dictionary or a list and removes keys specified by 'keys' parameter
243 @param data: dictionary or OrderedDict that will be modified.
244 Object is replaced in place
245 @type data: dictionary, OrderedDict
246 @param keys: array indicating the path to remove.
247 e.g. ['businessHours.headings[2]', 'businessHours.values.name']
248 @type keys: list
249 """
250 for key in keys:
251 JsonHelper._remove_key(data, key.split('.'), top_level)
252
253 @staticmethod
255 """
256 Takes a dictionary or a list and removes keys specified by 'keys' parameter
257 @param data: dictionary or OrderedDict that will be modified.
258 Object is replaced in place
259 @type data: dictionary, OrderedDict
260 @param key: array indicating the path to remove. e.g. ['businessHours', 'headings[2]']
261 @type key: list
262 """
263 orig_key = None
264
265
266 if type(key) is not list:
267 raise RuntimeError("Invalid argument key. Was expecting a list")
268
269 if not (type(data) in (dict, list) or isinstance(data, OrderedDict)):
270 raise RuntimeError("Invalid argument data. Was expecting a dict or OrderedDict")
271
272 if top_level:
273
274 orig_key = list(key)
275
276
277 match = re.search("^\*\[(.*)\]$", key[0])
278 if match:
279 try:
280 key[0] = int(match.group(1))
281 except ValueError:
282 raise RuntimeError("Index '%s' is not a valid integer" % match.group(2))
283
284
285
286 new_key = list()
287
288 for i, value in enumerate(key):
289 match = re.search("(^.+)\[(.+)\]$", str(value))
290 if match:
291 try:
292 new_key.append(match.group(1))
293 new_key.append(int(match.group(2)))
294 except ValueError:
295 raise RuntimeError("Index '%s' is not a valid integer" % match.group(2))
296
297 else:
298 new_key.append(value)
299
300 key = list(new_key)
301
302 if type(data) is list:
303
304 if type(key[0]) is int:
305 index = key.pop(0)
306
307 if len(key) == 0:
308
309 data.pop(index)
310 else:
311
312 JsonHelper._remove_key(data[index], key, top_level=False)
313
314 else:
315 raise RuntimeError("Key '%s' is not valid for the given JSON object" % key)
316 elif type(data) is dict or isinstance(data, OrderedDict):
317
318 for k in data:
319 if k == key[0]:
320 key.pop(0)
321
322 if len(key) == 0:
323
324 del(data[k])
325 else:
326
327 JsonHelper._remove_key(data[k], key, top_level=False)
328
329
330 break
331 else:
332 raise RuntimeError("Element %s is not allowed in a JSON response" % type(data))
333
334 if len(key) > 0:
335
336
337 warn("Key '%s' does not represent a valid element" % '.'.join(orig_key))
338
339 if top_level:
340
341
342 key = list(orig_key)
343
344 @staticmethod
346 """"
347 Applies a JSON path to a JSON string and returns the resulting node
348 @param data:
349 @type data: str, dict
350 """
351
352 if isinstance(data, basestring):
353 try:
354 json_data = json.loads(data)
355 except ValueError:
356 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
357 elif type(data) is dict:
358
359 json_data = data
360 else:
361 raise BadJsonFormatError("Parameter doesn't represent a valid JSON object: %s" % item)
362
363 jsonpath_expr = jsonpath_rw.parse(json_path)
364 result = []
365 for i in jsonpath_expr.find(json_data):
366 result.append(i.value)
367
368 return result
369