Package tlib :: Package base :: Module JsonHelper
[hide private]
[frames] | no frames]

Source Code for Module tlib.base.JsonHelper

  1  import copy 
  2  import re 
  3  import difflib 
  4  import json 
  5  import pytest 
  6  from collections import OrderedDict 
  7  from warnings import warn 
  8  from tlib.base.TestHelper import sort_list, sort_dict 
  9  import jsonpath_rw 
 10  from tlib.base.ExceptionHelper import BadJsonFormatError 
 11  from datadiff import diff 
12 13 14 # noinspection PyUnresolvedReferences 15 -class JsonHelper(object):
16 """ 17 Functions for modifying and validating JSON data 18 """ 19 20 @staticmethod
21 - def assert_json_equals(json1, json2, ignore_order=True, exclude_keys=list()):
22 """ 23 Compares two JSON values\n 24 If parameter validate_order is True, position of keys is taken into account during validation\n 25 If json1 or json2 are strings, it must be a valid JSON string 26 27 @param json1: String or dictionary to compare 28 @type json1: str, dict 29 @param json2: String or dictionary to compare 30 @type json2: str, dict 31 @param ignore_order: If true, order of keys is not validated 32 @type ignore_order: bool 33 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1' 34 @type exclude_keys: list 35 @raise AssertionError: JSON values doesn't match 36 """ 37 #Make copies of JSON objects so they are not modified 38 j1 = copy.deepcopy(json1) 39 j2 = copy.deepcopy(json2) 40 41 if ignore_order: 42 JsonHelper._validate_json_ignoring_order(j1, j2, exclude_keys) 43 else: 44 JsonHelper._validate_json_with_order(j1, j2, exclude_keys)
45 46 @staticmethod
47 - def _validate_json_ignoring_order(json1, json2, exclude_keys):
48 """ 49 Compares two JSON values, ignoring position of keys\n 50 If json1 or json2 are strings, it must be a valid JSON string 51 @param json1: String or dictionary to compare 52 @type json1: str, dict 53 @param json2: String or dictionary to compare 54 @type json2: str, dict 55 @raise AssertionError: JSON values doesn't match 56 """ 57 items = [json1, json2] 58 59 for i, item in enumerate(items): 60 #If it's a string, convert to dictionary 61 if isinstance(item, basestring): 62 try: 63 item = json.loads(item) 64 except ValueError: 65 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item) 66 67 #make sure any lists inside the JSON value are sorted, so diff will not fail 68 if type(item) is dict: 69 #Sort items inside lists 70 item = sort_dict(item) 71 elif type(item) is list: 72 #Sort items inside lists 73 item = sort_list(item) 74 elif type(item) is tuple: 75 pass 76 else: 77 raise RuntimeError("Parameter doesn't represent a valid JSON object: %s" % item) 78 79 if type(item) in (dict, list): 80 #Delete unwanted keys 81 JsonHelper.remove_keys(item, exclude_keys) 82 83 items[i] = item 84 85 JsonHelper.assert_equal(items[0], items[1])
86 87 @staticmethod
88 - def _validate_json_with_order(json1, json2, exclude_keys=list()):
89 """ 90 Compares two JSON values, taking into account key order 91 If json1 or json2 are strings, it must be a valid JSON string 92 @param json1: String to compare 93 @type json1: str 94 @param json2: String to compare 95 @type json2: str 96 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1' 97 @type exclude_keys: list 98 @raise AssertionError: JSON values doesn't match 99 """ 100 101 def object_pairs_hook(values): 102 """ 103 Method that stores objects in an OrderedDict so comparison takes into account key order 104 """ 105 return OrderedDict(values)
106 107 items = [json1, json2] 108 for i, item in enumerate(items): 109 if not isinstance(item, basestring): 110 raise RuntimeError("Only strings are allowed when validating key order") 111 112 # By default json.loads doesn't care about key order. Let's provide 113 # an object_pairs_hook function to ensure key order is kept 114 try: 115 item = json.loads(item, object_pairs_hook=object_pairs_hook) 116 except ValueError: 117 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item) 118 119 #Delete unwanted keys 120 JsonHelper.remove_keys(item, exclude_keys) 121 122 items[i] = item 123 124 #Do validation 125 if items[0] != items[1]: 126 json1 = json.dumps(items[0], indent=3).splitlines(True) 127 json2 = json.dumps(items[1], indent=3).splitlines(True) 128 diff = difflib.unified_diff(json1, json2) 129 diff_txt = "".join(diff) 130 131 raise AssertionError(r"JSON values didn't match\nValue1:\n%s\n\nValue2:\n%s\n\nDiff:\n%s""" % 132 ("".join(json1), "".join(json2), diff_txt))
133 134 @staticmethod
135 - def assert_equal(dictA, dictB):
136 """ 137 Takes two dictionaries, compares and displays any distinct keys in both, matching keys but different values, matching key/values 138 @param dictA: dictionary or tuple (for e.g. expected dictionary) 139 @type dictA: dictionary 140 @param dictB: dictionary or tuple (for e.g. dictionary captured at run time) 141 @type dictA: dictionary 142 e.g. Tuple: dictA = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['en']}, {'drink': ['water'], 'weather': ['rainy']}, {'device': ['iPad']} 143 dictB = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['fr'], 'color':['blue']}, {'drink': ['alcohol'], 'weather': ['rainy']}, {'device': ['iPad']} 144 Output: (prints only the non matching dictionaries with the differences) 145 > assert False, '\n'.join(nonmatching_dict) 146 E AssertionError: 147 E 148 E Dictionary_1 149 E distinct_tags_in_dict_1A: None 150 E distinct_tags_in_dict_1B: {color:['blue']} 151 E nonmatching_tags_in_two_dicts:{ln:['en']} & {ln:['fr']} respectively 152 E matching_tags_in_two_dicts:{city:['montreal']} 153 E 154 E 155 E Dictionary_2 156 E distinct_tags_in_dict_2A: None 157 E distinct_tags_in_dict_2B: None 158 E nonmatching_tags_in_two_dicts:{drink:['water']} & {drink:['alcohol']} respectively 159 E matching_tags_in_two_dicts:{weather:['rainy']} 160 """ 161 162 nonmatching_dict = [] 163 matching_dict = [] 164 165 distinct_tags_in_dictA = [] 166 distinct_tags_in_dictB = [] 167 matching_tags_in_two_dicts = [] 168 nonmatching_tags_in_two_dicts = [] 169 170 myflag = [] 171 i = 0 172 tot_dict = 0 173 174 # calculates the number of dictionaries in expected dictionary/tuple 175 if type(dictA) is dict: 176 tot_dict = 1 177 dictA = eval('[%s]' % dictA) 178 dictB = eval('[%s]' % dictB) 179 elif type(dictA) is tuple or list: 180 tot_dict = len(dictA) 181 182 # compares two dictionaries and prints differences in detail 183 for i in range(tot_dict): 184 differences = diff(dictA[i], dictB[i]) 185 for dif in differences.diffs: 186 if dif[0] == 'context_end_container': 187 break 188 else: 189 # dif = {tuple}('insert', ['cd': ['24']]) 190 key = dif[1][0][0] 191 expected_val = dictA[i].get(key) 192 captured_val = dictB[i].get(key) 193 194 # Captures tags that exists only in Dictionary B 195 if dif[0] == 'insert': 196 # No validation here, just captures the distinct tags in Dictionary B 197 distinct_tags_in_dictB.append("{%s:%s}" %(key, captured_val)) 198 199 # Captures tags that exists only in Dictionary A 200 elif dif[0] == 'delete': 201 myflag.append("False") 202 distinct_tags_in_dictA.append("{%s:%s}" %(key, expected_val)) 203 204 elif dif[0] == 'equal': 205 # Captures tags that do not match in Dictionary A and Dictionary B 206 if expected_val != captured_val: 207 myflag.append("False") 208 nonmatching_tags_in_two_dicts.append("{%s:%s} & {%s:%s} respectively" %(key, expected_val,key, captured_val)) 209 210 # Captures tags (key/value) that matches in Dictionary A and Dictionary B 211 else: 212 myflag.append("True") 213 matching_tags_in_two_dicts.append("{%s:%s}" % (key, expected_val)) 214 215 # Captures tags (key/value) that do not match in Dictionary A and Dictionary B 216 elif expected_val != captured_val: 217 myflag.append("False") 218 nonmatching_tags_in_two_dicts.append("{%s:%s}/{%s:%s}" %(key, expected_val, key, captured_val)) 219 220 if (distinct_tags_in_dictA and distinct_tags_in_dictB) or (nonmatching_tags_in_two_dicts) != [] and ("False" in myflag): 221 if distinct_tags_in_dictA ==[]: distinct_tags_in_dictA.append("None") 222 if distinct_tags_in_dictB ==[]: distinct_tags_in_dictB.append("None") 223 if nonmatching_tags_in_two_dicts ==[]: nonmatching_tags_in_two_dicts.append("None") 224 if matching_tags_in_two_dicts ==[]: matching_tags_in_two_dicts.append("None") 225 nonmatching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "distinct_tags_in_dict_%dA: " % int(i+1) + ','.join(distinct_tags_in_dictA) + '\n' + "distinct_tags_in_dict_%dB: " % int(i+1) + ','.join(distinct_tags_in_dictB) + '\n' + "nonmatching_tags_in_two_dicts:" + ','.join(nonmatching_tags_in_two_dicts) + '\n' + "matching_tags_in_two_dicts:" + ','.join(matching_tags_in_two_dicts)) 226 elif "True" in myflag: 227 matching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "matching_tags_in_two_dicts: " + ','.join(matching_tags_in_two_dicts)) 228 229 # clears content of the temp dictionaries 230 distinct_tags_in_dictA = [] 231 distinct_tags_in_dictB = [] 232 matching_tags_in_two_dicts = [] 233 nonmatching_tags_in_two_dicts = [] 234 235 if "False" in myflag: 236 assert False, '\n'.join(nonmatching_dict)
237 238 239 @staticmethod
240 - def remove_keys(data, keys, top_level=True):
241 """ 242 Takes a dictionary or a list and removes keys specified by 'keys' parameter 243 @param data: dictionary or OrderedDict that will be modified. 244 Object is replaced in place 245 @type data: dictionary, OrderedDict 246 @param keys: array indicating the path to remove. 247 e.g. ['businessHours.headings[2]', 'businessHours.values.name'] 248 @type keys: list 249 """ 250 for key in keys: 251 JsonHelper._remove_key(data, key.split('.'), top_level)
252 253 @staticmethod
254 - def _remove_key(data, key, top_level=True):
255 """ 256 Takes a dictionary or a list and removes keys specified by 'keys' parameter 257 @param data: dictionary or OrderedDict that will be modified. 258 Object is replaced in place 259 @type data: dictionary, OrderedDict 260 @param key: array indicating the path to remove. e.g. ['businessHours', 'headings[2]'] 261 @type key: list 262 """ 263 orig_key = None 264 265 #Parameter validation 266 if type(key) is not list: 267 raise RuntimeError("Invalid argument key. Was expecting a list") 268 269 if not (type(data) in (dict, list) or isinstance(data, OrderedDict)): 270 raise RuntimeError("Invalid argument data. Was expecting a dict or OrderedDict") 271 272 if top_level: 273 #Create a copy of the key object so we don't modify the original 274 orig_key = list(key) 275 276 #Check if first element in the keys has syntax '*' and change it to be able to match the right values 277 match = re.search("^\*\[(.*)\]$", key[0]) 278 if match: 279 try: 280 key[0] = int(match.group(1)) 281 except ValueError: 282 raise RuntimeError("Index '%s' is not a valid integer" % match.group(2)) 283 284 # split indexed items in two, only first time function is called 285 # eg. ["node1", "node[2]"] => ["node1", "node", 2] 286 new_key = list() 287 288 for i, value in enumerate(key): 289 match = re.search("(^.+)\[(.+)\]$", str(value)) 290 if match: 291 try: 292 new_key.append(match.group(1)) 293 new_key.append(int(match.group(2))) 294 except ValueError: 295 raise RuntimeError("Index '%s' is not a valid integer" % match.group(2)) 296 297 else: 298 new_key.append(value) 299 300 key = list(new_key) 301 302 if type(data) is list: 303 #check if next key is an index, otherwise fail 304 if type(key[0]) is int: 305 index = key.pop(0) 306 307 if len(key) == 0: 308 #Found the key 309 data.pop(index) 310 else: 311 #Still need to find children nodes 312 JsonHelper._remove_key(data[index], key, top_level=False) 313 314 else: 315 raise RuntimeError("Key '%s' is not valid for the given JSON object" % key) 316 elif type(data) is dict or isinstance(data, OrderedDict): 317 #Validate 318 for k in data: 319 if k == key[0]: 320 key.pop(0) 321 322 if len(key) == 0: 323 #Found the key 324 del(data[k]) 325 else: 326 #Still need to find children nodes 327 JsonHelper._remove_key(data[k], key, top_level=False) 328 329 # Don't need to continue iterating. Node was already found 330 break 331 else: 332 raise RuntimeError("Element %s is not allowed in a JSON response" % type(data)) 333 334 if len(key) > 0: 335 # Not all keys were found. Can't fail here because it would make impossible testing the scenario where 336 # a JSON payload has a key and other doesn't 337 warn("Key '%s' does not represent a valid element" % '.'.join(orig_key)) 338 339 if top_level: 340 #Restore key variable 341 # noinspection PyUnusedLocal 342 key = list(orig_key)
343 344 @staticmethod
345 - def get_elements(data, json_path):
346 """" 347 Applies a JSON path to a JSON string and returns the resulting node 348 @param data: 349 @type data: str, dict 350 """ 351 #If it's a string, convert to dictionary 352 if isinstance(data, basestring): 353 try: 354 json_data = json.loads(data) 355 except ValueError: 356 raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item) 357 elif type(data) is dict: 358 #Sort items inside lists 359 json_data = data 360 else: 361 raise BadJsonFormatError("Parameter doesn't represent a valid JSON object: %s" % item) 362 363 jsonpath_expr = jsonpath_rw.parse(json_path) 364 result = [] 365 for i in jsonpath_expr.find(json_data): 366 result.append(i.value) 367 368 return result
369