tlib.base.JsonHelper

1 import re 2 import difflib 3 import json 4 import pytest 5 from collections import OrderedDict 6 from warnings import warn 7 from tlib.base.TestHelper import sort_list, sort_dict 8 import jsonpath_rw 9 from tlib.base.ExceptionHelper import BadJsonFormatError 10 from datadiff import diff

11 12 13 # noinspection PyUnresolvedReferences 14 -class JsonHelper(object):

15 """ 16 Functions for modifying and validating JSON data 17 """ 18 19 @staticmethod

20 - def assert_json_equals(json1, json2, ignore_order=True, exclude_keys=list()):

21 """ 22 Compares two JSON values\n 23 If parameter validate_order is True, position of keys is taken into account during validation\n 24 If json1 or json2 are strings, it must be a valid JSON string 25 26 @param json1: String or dictionary to compare 27 @type json1: str, dict 28 @param json2: String or dictionary to compare 29 @type json2: str, dict 30 @param ignore_order: If true, order of keys is not validated 31 @type ignore_order: bool 32 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1' 33 @type exclude_keys: list 34 @raise AssertionError: JSON values doesn't match 35 """ 36 if ignore_order: 37 JsonHelper._validate_json_ignoring_order(json1, json2, exclude_keys) 38 else: 39 JsonHelper._validate_json_with_order(json1, json2, exclude_keys)

40 41 @staticmethod

42 - def _validate_json_ignoring_order(json1, json2, exclude_keys):

43 """ 44 Compares two JSON values, ignoring position of keys\n 45 If json1 or json2 are strings, it must be a valid JSON string 46 @param json1: String or dictionary to compare 47 @type json1: str, dict 48 @param json2: String or dictionary to compare 49 @type json2: str, dict 50 @raise AssertionError: JSON values doesn't match 51 """ 52 items = [json1, json2] 53 54 for i, item in enumerate(items): 55 #If it's a string, convert to dictionary 56 if isinstance(item, basestring): 57 try: 58 item = json.loads(item) 59 except ValueError: 60 pytest.fail("Value doesn't represent a valid JSON string\n%s" % item) 61 62 #make sure any lists inside the JSON value are sorted, so diff will not fail 63 if type(item) is dict: 64 #Sort items inside lists 65 item = sort_dict(item) 66 elif type(item) is list: 67 #Sort items inside lists 68 item = sort_list(item) 69 elif type(item) is tuple: 70 pass 71 else: 72 pytest.fail("Parameter doesn't represent a valid JSON object: %s" % item) 73 74 if type(item) in (dict, list): 75 #Delete unwanted keys 76 JsonHelper.remove_keys(item, exclude_keys) 77 78 items[i] = item 79 80 JsonHelper.assert_equal(items[0], items[1])

81 82 @staticmethod

83 - def _validate_json_with_order(json1, json2, exclude_keys=list()):

84 """ 85 Compares two JSON values, taking into account key order 86 If json1 or json2 are strings, it must be a valid JSON string 87 @param json1: String to compare 88 @type json1: str 89 @param json2: String to compare 90 @type json2: str 91 @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1' 92 @type exclude_keys: list 93 @raise AssertionError: JSON values doesn't match 94 """ 95 96 def object_pairs_hook(values): 97 """ 98 Method that stores objects in an OrderedDict so comparison takes into account key order 99 """ 100 return OrderedDict(values)

101 102 items = [json1, json2] 103 for i, item in enumerate(items): 104 if not isinstance(item, basestring): 105 pytest.fail("Only strings are allowed when validating key order") 106 107 # By default json.loads doesn't care about key order. Let's provide 108 # an object_pairs_hook function to ensure key order is kept 109 try: 110 item = json.loads(item, object_pairs_hook=object_pairs_hook) 111 except ValueError: 112 pytest.fail("Value doesn't represent a valid JSON string\n%s" % item) 113 114 #Delete unwanted keys 115 JsonHelper.remove_keys(item, exclude_keys) 116 117 items[i] = item 118 119 #Do validation 120 if items[0] != items[1]: 121 json1 = json.dumps(items[0], indent=3).splitlines(True) 122 json2 = json.dumps(items[1], indent=3).splitlines(True) 123 diff = difflib.unified_diff(json1, json2) 124 diff_txt = "".join(diff) 125 126 raise AssertionError(r"JSON values didn't match\nValue1:\n%s\n\nValue2:\n%s\n\nDiff:\n%s""" % 127 ("".join(json1), "".join(json2), diff_txt))

128 129 @staticmethod

130 - def assert_equal(dictA, dictB):

131 """ 132 Takes two dictionaries, compares and displays any distinct keys in both, matching keys but different values, matching key/values 133 @param dictA: dictionary or tuple (for e.g. expected dictionary) 134 @type dictA: dictionary 135 @param dictB: dictionary or tuple (for e.g. dictionary captured at run time) 136 @type dictA: dictionary 137 e.g. Tuple: dictA = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['en']}, {'drink': ['water'], 'weather': ['rainy']}, {'device': ['iPad']} 138 dictB = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['fr'], 'color':['blue']}, {'drink': ['alcohol'], 'weather': ['rainy']}, {'device': ['iPad']} 139 Output: (prints only the non matching dictionaries with the differences) 140 > assert False, '\n'.join(nonmatching_dict) 141 E AssertionError: 142 E 143 E Dictionary_1 144 E distinct_tags_in_dict_1A: None 145 E distinct_tags_in_dict_1B: {color:['blue']} 146 E nonmatching_tags_in_two_dicts:{ln:['en']} & {ln:['fr']} respectively 147 E matching_tags_in_two_dicts:{city:['montreal']} 148 E 149 E 150 E Dictionary_2 151 E distinct_tags_in_dict_2A: None 152 E distinct_tags_in_dict_2B: None 153 E nonmatching_tags_in_two_dicts:{drink:['water']} & {drink:['alcohol']} respectively 154 E matching_tags_in_two_dicts:{weather:['rainy']} 155 """ 156 157 nonmatching_dict = [] 158 matching_dict = [] 159 160 distinct_tags_in_dictA = [] 161 distinct_tags_in_dictB = [] 162 matching_tags_in_two_dicts = [] 163 nonmatching_tags_in_two_dicts = [] 164 165 myflag = [] 166 i = 0 167 tot_dict = 0 168 169 # calculates the number of dictionaries in expected dictionary/tuple 170 if type(dictA) is dict: 171 tot_dict = 1 172 dictA = eval('[%s]' % dictA) 173 dictB = eval('[%s]' % dictB) 174 elif type(dictA) is tuple or list: 175 tot_dict = len(dictA) 176 177 # compares two dictionaries and prints differences in detail 178 for i in range(tot_dict): 179 differences = diff(dictA[i], dictB[i]) 180 for dif in differences.diffs: 181 if dif[0] == 'context_end_container': 182 break 183 else: 184 # dif = {tuple}('insert', ['cd': ['24']]) 185 key = dif[1][0][0] 186 expected_val = dictA[i].get(key) 187 captured_val = dictB[i].get(key) 188 189 # Captures tags that exists only in Dictionary B 190 if dif[0] == 'insert': 191 # No validation here, just captures the distinct tags in Dictionary B 192 distinct_tags_in_dictB.append("{%s:%s}" %(key, captured_val)) 193 194 # Captures tags that exists only in Dictionary A 195 elif dif[0] == 'delete': 196 myflag.append("False") 197 distinct_tags_in_dictA.append("{%s:%s}" %(key, expected_val)) 198 199 elif dif[0] == 'equal': 200 # Captures tags that do not match in Dictionary A and Dictionary B 201 if expected_val != captured_val: 202 myflag.append("False") 203 nonmatching_tags_in_two_dicts.append("{%s:%s} & {%s:%s} respectively" %(key, expected_val,key, captured_val)) 204 205 # Captures tags (key/value) that matches in Dictionary A and Dictionary B 206 else: 207 myflag.append("True") 208 matching_tags_in_two_dicts.append("{%s:%s}" % (key, expected_val)) 209 210 # Captures tags (key/value) that do not match in Dictionary A and Dictionary B 211 elif expected_val != captured_val: 212 myflag.append("False") 213 nonmatching_tags_in_two_dicts.append("{%s:%s}/{%s:%s}" %(key, expected_val, key, captured_val)) 214 215 if (distinct_tags_in_dictA and distinct_tags_in_dictB) or (nonmatching_tags_in_two_dicts) != [] and ("False" in myflag): 216 if distinct_tags_in_dictA ==[]: distinct_tags_in_dictA.append("None") 217 if distinct_tags_in_dictB ==[]: distinct_tags_in_dictB.append("None") 218 if nonmatching_tags_in_two_dicts ==[]: nonmatching_tags_in_two_dicts.append("None") 219 if matching_tags_in_two_dicts ==[]: matching_tags_in_two_dicts.append("None") 220 nonmatching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "distinct_tags_in_dict_%dA: " % int(i+1) + ','.join(distinct_tags_in_dictA) + '\n' + "distinct_tags_in_dict_%dB: " % int(i+1) + ','.join(distinct_tags_in_dictB) + '\n' + "nonmatching_tags_in_two_dicts:" + ','.join(nonmatching_tags_in_two_dicts) + '\n' + "matching_tags_in_two_dicts:" + ','.join(matching_tags_in_two_dicts)) 221 elif "True" in myflag: 222 matching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "matching_tags_in_two_dicts: " + ','.join(matching_tags_in_two_dicts)) 223 224 # clears content of the temp dictionaries 225 distinct_tags_in_dictA = [] 226 distinct_tags_in_dictB = [] 227 matching_tags_in_two_dicts = [] 228 nonmatching_tags_in_two_dicts = [] 229 230 if "False" in myflag: 231 assert False, '\n'.join(nonmatching_dict)

232 233 234 @staticmethod

235 - def remove_keys(data, keys, top_level=True):

236 """ 237 Takes a dictionary or a list and removes keys specified by 'keys' parameter 238 @param data: dictionary or OrderedDict that will be modified. 239 Object is replaced in place 240 @type data: dictionary, OrderedDict 241 @param keys: array indicating the path to remove. 242 e.g. ['businessHours.headings[2]', 'businessHours.values.name'] 243 @type keys: list 244 """ 245 for key in keys: 246 JsonHelper._remove_key(data, key.split('.'), top_level)

247 248 @staticmethod

249 - def _remove_key(data, key, top_level=True):

250 """ 251 Takes a dictionary or a list and removes keys specified by 'keys' parameter 252 @param data: dictionary or OrderedDict that will be modified. 253 Object is replaced in place 254 @type data: dictionary, OrderedDict 255 @param key: array indicating the path to remove. e.g. ['businessHours', 'headings[2]'] 256 @type key: list 257 """ 258 orig_key = None 259 260 #Parameter validation 261 if type(key) is not list: 262 pytest.fail("Invalid argument key. Was expecting a list") 263 264 if not (type(data) in (dict, list) or isinstance(data, OrderedDict)): 265 pytest.fail("Invalid argument data. Was expecting a dict or OrderedDict") 266 267 if top_level: 268 #Create a copy of the key object so we don't modify the original 269 orig_key = list(key) 270 271 #Check if first element in the keys has syntax '*' and change it to be able to match the right values 272 match = re.search("^\*\[(.*)\]$", key[0]) 273 if match: 274 try: 275 key[0] = int(match.group(1)) 276 except ValueError: 277 pytest.fail("Index '%s' is not a valid integer" % match.group(2)) 278 279 # split indexed items in two, only first time function is called 280 # eg. ["node1", "node[2]"] => ["node1", "node", 2] 281 new_key = list() 282 283 for i, value in enumerate(key): 284 match = re.search("(^.+)\[(.+)\]$", str(value)) 285 if match: 286 try: 287 new_key.append(match.group(1)) 288 new_key.append(int(match.group(2))) 289 except ValueError: 290 pytest.fail("Index '%s' is not a valid integer" % match.group(2)) 291 292 else: 293 new_key.append(value) 294 295 key = list(new_key) 296 297 if type(data) is list: 298 #check if next key is an index, otherwise fail 299 if type(key[0]) is int: 300 index = key.pop(0) 301 302 if len(key) == 0: 303 #Found the key 304 data.pop(index) 305 else: 306 #Still need to find children nodes 307 JsonHelper._remove_key(data[index], key, top_level=False) 308 309 else: 310 pytest.fail("Key '%s' is not valid for the given JSON object" % key) 311 elif type(data) is dict or isinstance(data, OrderedDict): 312 #Validate 313 for k in data: 314 if k == key[0]: 315 key.pop(0) 316 317 if len(key) == 0: 318 #Found the key 319 del(data[k]) 320 else: 321 #Still need to find children nodes 322 JsonHelper._remove_key(data[k], key, top_level=False) 323 324 # Don't need to continue iterating. Node was already found 325 break 326 else: 327 pytest.fail("Element %s is not allowed in a JSON response" % type(data)) 328 329 if len(key) > 0: 330 # Not all keys were found. Can't fail here because it would make impossible testing the scenario where 331 # a JSON payload has a key and other doesn't 332 warn("Key '%s' does not represent a valid element" % '.'.join(orig_key)) 333 334 if top_level: 335 #Restore key variable 336 # noinspection PyUnusedLocal 337 key = list(orig_key)

338 339 @staticmethod

340 - def get_elements(data, json_path):

341 """" 342 Applies a JSON path to a JSON string and returns the resulting node 343 @param data: 344 @type data: str, dict 345 """ 346 #If it's a string, convert to dictionary 347 if isinstance(data, basestring): 348 try: 349 json_data = json.loads(data) 350 except ValueError: 351 pytest.fail("Value doesn't represent a valid JSON string\n%s" % item) 352 elif type(data) is dict: 353 #Sort items inside lists 354 json_data = data 355 else: 356 raise BadJsonFormatError("Parameter doesn't represent a valid JSON object: %s" % item) 357 358 jsonpath_expr = jsonpath_rw.parse(json_path) 359 result = [] 360 for i in jsonpath_expr.find(json_data): 361 result.append(i.value) 362 363 return result

364

Source Code for Module tlib.base.JsonHelper