Package tlib :: Package base :: Module JsonHelper |
[hide private]
[frames] | no frames]

# Source Code for Module tlib.base.JsonHelper

      1  import copy
      2  import re
      3  import difflib
      4  import json
      5  import pytest
      6  from collections import OrderedDict
      7  from warnings import warn
      8  from tlib.base.TestHelper import sort_list, sort_dict
      9  import jsonpath_rw
     10  from tlib.base.ExceptionHelper import BadJsonFormatError
     11  from datadiff import diff
     12
     13
     14  # noinspection PyUnresolvedReferences
     15 -class JsonHelper(object):
     16      """
     17      Functions for modifying and validating JSON data
     18      """
     19
     20      @staticmethod
     21 -    def assert_json_equals(json1, json2, ignore_order=True, exclude_keys=list()):
     22          """
     23          Compares two JSON values\n
     24          If parameter validate_order is True, position of keys is taken into account during validation\n
     25          If json1 or json2 are strings, it must be a valid JSON string
     26
     27          @param json1: String or dictionary to compare
     28          @type json1: str, dict
     29          @param json2: String or dictionary to compare
     30          @type json2: str, dict
     31          @param ignore_order: If true, order of keys is not validated
     32          @type ignore_order: bool
     33          @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1'
     34          @type exclude_keys: list
     35          @raise AssertionError: JSON values doesn't match
     36          """
     37          #Make copies of JSON objects so they are not modified
     38          j1 = copy.deepcopy(json1)
     39          j2 = copy.deepcopy(json2)
     40
     41          if ignore_order:
     42              JsonHelper._validate_json_ignoring_order(j1, j2, exclude_keys)
     43          else:
     44              JsonHelper._validate_json_with_order(j1, j2, exclude_keys)
     45
     46      @staticmethod
     47 -    def _validate_json_ignoring_order(json1, json2, exclude_keys):
     48          """
     49          Compares two JSON values, ignoring position of keys\n
     50          If json1 or json2 are strings, it must be a valid JSON string
     51          @param json1: String or dictionary to compare
     52          @type json1: str, dict
     53          @param json2: String or dictionary to compare
     54          @type json2: str, dict
     55          @raise AssertionError: JSON values doesn't match
     56          """
     57          items = [json1, json2]
     58
     59          for i, item in enumerate(items):
     60              #If it's a string, convert to dictionary
     61              if isinstance(item, basestring):
     62                  try:
     63                      item = json.loads(item)
     64                  except ValueError:
     65                      raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
     66
     67              #make sure any lists inside the JSON value are sorted, so diff will not fail
     68              if type(item) is dict:
     69                  #Sort items inside lists
     70                  item = sort_dict(item)
     71              elif type(item) is list:
     72                  #Sort items inside lists
     73                  item = sort_list(item)
     74              elif type(item) is tuple:
     75                  pass
     76              else:
     77                  raise RuntimeError("Parameter doesn't represent a valid JSON object: %s" % item)
     78
     79              if type(item) in (dict, list):
     80                  #Delete unwanted keys
     81                  JsonHelper.remove_keys(item, exclude_keys)
     82
     83                  items[i] = item
     84
     85          JsonHelper.assert_equal(items[0], items[1])
     86
     87      @staticmethod
     88 -    def _validate_json_with_order(json1, json2, exclude_keys=list()):
     89          """
     90          Compares two JSON values, taking into account key order
     91          If json1 or json2 are strings, it must be a valid JSON string
     92          @param json1: String to compare
     93          @type json1: str
     94          @param json2: String to compare
     95          @type json2: str
     96          @param exclude_keys: Keys to exclude from comparison, in the format 'key2.key3[2].key1'
     97          @type exclude_keys: list
     98          @raise AssertionError: JSON values doesn't match
     99          """
    100
    101          def object_pairs_hook(values):
    102              """
    103              Method that stores objects in an OrderedDict so comparison takes into account key order
    104              """
    105              return OrderedDict(values)
    106
    107          items = [json1, json2]
    108          for i, item in enumerate(items):
    109              if not isinstance(item, basestring):
    110                  raise RuntimeError("Only strings are allowed when validating key order")
    111
    112              # By default json.loads doesn't care about key order. Let's provide
    113              # an object_pairs_hook function to ensure key order is kept
    114              try:
    115                  item = json.loads(item, object_pairs_hook=object_pairs_hook)
    116              except ValueError:
    117                  raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
    118
    119              #Delete unwanted keys
    120              JsonHelper.remove_keys(item, exclude_keys)
    121
    122              items[i] = item
    123
    124          #Do validation
    125          if items[0] != items[1]:
    126              json1 = json.dumps(items[0], indent=3).splitlines(True)
    127              json2 = json.dumps(items[1], indent=3).splitlines(True)
    128              diff = difflib.unified_diff(json1, json2)
    129              diff_txt = "".join(diff)
    130
    131              raise AssertionError(r"JSON values didn't match\nValue1:\n%s\n\nValue2:\n%s\n\nDiff:\n%s""" %
    132                                   ("".join(json1), "".join(json2), diff_txt))
    133
    134      @staticmethod
    135 -    def assert_equal(dictA, dictB):
    136          """
    137          Takes two dictionaries, compares and displays any distinct keys in both, matching keys but different values, matching key/values
    138          @param dictA: dictionary or tuple (for e.g. expected dictionary)
    139          @type dictA: dictionary
    140          @param dictB: dictionary or tuple (for e.g. dictionary captured at run time)
    141          @type dictA: dictionary
    142          e.g. Tuple: dictA = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['en']}, {'drink': ['water'], 'weather': ['rainy']}, {'device': ['iPad']}
    143                      dictB = {Dictionary1, Dictionary2, Dictionary3} = {'city': ['montreal'], 'ln': ['fr'], 'color':['blue']}, {'drink': ['alcohol'], 'weather': ['rainy']}, {'device': ['iPad']}
    144              Output: (prints only the non matching dictionaries with the differences)
    145              >           assert False, '\n'.join(nonmatching_dict)
    146              E           AssertionError:
    147              E
    148              E           Dictionary_1
    149              E           distinct_tags_in_dict_1A: None
    150              E           distinct_tags_in_dict_1B: {color:['blue']}
    151              E           nonmatching_tags_in_two_dicts:{ln:['en']} & {ln:['fr']} respectively
    152              E           matching_tags_in_two_dicts:{city:['montreal']}
    153              E
    154              E
    155              E           Dictionary_2
    156              E           distinct_tags_in_dict_2A: None
    157              E           distinct_tags_in_dict_2B: None
    158              E           nonmatching_tags_in_two_dicts:{drink:['water']} & {drink:['alcohol']} respectively
    159              E           matching_tags_in_two_dicts:{weather:['rainy']}
    160          """
    161
    162          nonmatching_dict = []
    163          matching_dict = []
    164
    165          distinct_tags_in_dictA = []
    166          distinct_tags_in_dictB = []
    167          matching_tags_in_two_dicts = []
    168          nonmatching_tags_in_two_dicts = []
    169
    170          myflag = []
    171          i = 0
    172          tot_dict = 0
    173
    174          # calculates the number of dictionaries in expected dictionary/tuple
    175          if type(dictA) is dict:
    176              tot_dict = 1
    177              dictA = eval('[%s]' % dictA)
    178              dictB = eval('[%s]' % dictB)
    179          elif type(dictA) is tuple or list:
    180              tot_dict = len(dictA)
    181
    182          # compares two dictionaries and prints differences in detail
    183          for i in range(tot_dict):
    184              differences = diff(dictA[i], dictB[i])
    185              for dif in differences.diffs:
    186                  if dif[0] == 'context_end_container':
    187                      break
    188                  else:
    189                      # dif = {tuple}('insert', ['cd': ['24']])
    190                      key = dif[1][0][0]
    191                      expected_val = dictA[i].get(key)
    192                      captured_val = dictB[i].get(key)
    193
    194                      # Captures tags that exists only in Dictionary B
    195                      if dif[0] == 'insert':
    196                          # No validation here, just captures the distinct tags in Dictionary B
    197                          distinct_tags_in_dictB.append("{%s:%s}" %(key, captured_val))
    198
    199                      # Captures tags that exists only in Dictionary A
    200                      elif dif[0] == 'delete':
    201                          myflag.append("False")
    202                          distinct_tags_in_dictA.append("{%s:%s}" %(key, expected_val))
    203
    204                      elif dif[0] == 'equal':
    205                          # Captures tags that do not match in Dictionary A and Dictionary B
    206                          if expected_val != captured_val:
    207                              myflag.append("False")
    208                              nonmatching_tags_in_two_dicts.append("{%s:%s} & {%s:%s} respectively" %(key, expected_val,key, captured_val))
    209
    210                          # Captures tags (key/value) that matches in Dictionary A and Dictionary B
    211                          else:
    212                              myflag.append("True")
    213                              matching_tags_in_two_dicts.append("{%s:%s}" % (key, expected_val))
    214
    215                      # Captures tags (key/value) that do not match in Dictionary A and Dictionary B
    216                      elif expected_val != captured_val:
    217                          myflag.append("False")
    218                          nonmatching_tags_in_two_dicts.append("{%s:%s}/{%s:%s}" %(key, expected_val, key, captured_val))
    219
    220              if (distinct_tags_in_dictA and distinct_tags_in_dictB) or (nonmatching_tags_in_two_dicts) != [] and ("False" in myflag):
    221                  if distinct_tags_in_dictA ==[]: distinct_tags_in_dictA.append("None")
    222                  if distinct_tags_in_dictB ==[]: distinct_tags_in_dictB.append("None")
    223                  if nonmatching_tags_in_two_dicts ==[]: nonmatching_tags_in_two_dicts.append("None")
    224                  if matching_tags_in_two_dicts ==[]: matching_tags_in_two_dicts.append("None")
    225                  nonmatching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' +  "distinct_tags_in_dict_%dA: " % int(i+1) +  ','.join(distinct_tags_in_dictA) + '\n' + "distinct_tags_in_dict_%dB: " % int(i+1) + ','.join(distinct_tags_in_dictB)  + '\n' + "nonmatching_tags_in_two_dicts:" +  ','.join(nonmatching_tags_in_two_dicts) + '\n' + "matching_tags_in_two_dicts:" +  ','.join(matching_tags_in_two_dicts))
    226              elif "True" in myflag:
    227                  matching_dict.append('\n' + '\n' +"Dictionary_%d" % int(i+1) + '\n' + "matching_tags_in_two_dicts: " +  ','.join(matching_tags_in_two_dicts))
    228
    229              # clears content of the temp dictionaries
    230              distinct_tags_in_dictA = []
    231              distinct_tags_in_dictB = []
    232              matching_tags_in_two_dicts = []
    233              nonmatching_tags_in_two_dicts = []
    234
    235          if "False" in myflag:
    236              assert False, '\n'.join(nonmatching_dict)
    237
    238
    239      @staticmethod
    240 -    def remove_keys(data, keys, top_level=True):
    241          """
    242          Takes a dictionary or a list and removes keys specified by 'keys' parameter
    243          @param data: dictionary or OrderedDict that will be modified.
    244          Object is replaced in place
    245          @type data: dictionary, OrderedDict
    246          @param keys: array indicating the path to remove.
    247          e.g. ['businessHours.headings[2]', 'businessHours.values.name']
    248          @type keys: list
    249          """
    250          for key in keys:
    251              JsonHelper._remove_key(data, key.split('.'), top_level)
    252
    253      @staticmethod
    254 -    def _remove_key(data, key, top_level=True):
    255          """
    256          Takes a dictionary or a list and removes keys specified by 'keys' parameter
    257          @param data: dictionary or OrderedDict that will be modified.
    258          Object is replaced in place
    259          @type data: dictionary, OrderedDict
    260          @param key: array indicating the path to remove. e.g. ['businessHours', 'headings[2]']
    261          @type key: list
    262          """
    263          orig_key = None
    264
    265          #Parameter validation
    266          if type(key) is not list:
    267              raise RuntimeError("Invalid argument key. Was expecting a list")
    268
    269          if not (type(data) in (dict, list) or isinstance(data, OrderedDict)):
    270              raise RuntimeError("Invalid argument data. Was expecting a dict or OrderedDict")
    271
    272          if top_level:
    273              #Create a copy of the key object so we don't modify the original
    274              orig_key = list(key)
    275
    276              #Check if first element in the keys has syntax '*' and change it to be able to match the right values
    277              match = re.search("^\*\[(.*)\]$", key[0])
    278              if match:
    279                  try:
    280                      key[0] = int(match.group(1))
    281                  except ValueError:
    282                      raise RuntimeError("Index '%s' is not a valid integer" % match.group(2))
    283
    284              # split indexed items in two, only first time function is called
    285              # eg. ["node1", "node[2]"] => ["node1", "node", 2]
    286              new_key = list()
    287
    288              for i, value in enumerate(key):
    289                  match = re.search("(^.+)\[(.+)\]$", str(value))
    290                  if match:
    291                      try:
    292                          new_key.append(match.group(1))
    293                          new_key.append(int(match.group(2)))
    294                      except ValueError:
    295                          raise RuntimeError("Index '%s' is not a valid integer" % match.group(2))
    296
    297                  else:
    298                      new_key.append(value)
    299
    300              key = list(new_key)
    301
    302          if type(data) is list:
    303              #check if next key is an index, otherwise fail
    304              if type(key[0]) is int:
    305                  index = key.pop(0)
    306
    307                  if len(key) == 0:
    308                      #Found the key
    309                      data.pop(index)
    310                  else:
    311                      #Still need to find children nodes
    312                      JsonHelper._remove_key(data[index], key, top_level=False)
    313
    314              else:
    315                  raise RuntimeError("Key '%s' is not valid for the given JSON object" % key)
    316          elif type(data) is dict or isinstance(data, OrderedDict):
    317              #Validate
    318              for k in data:
    319                  if k == key[0]:
    320                      key.pop(0)
    321
    322                      if len(key) == 0:
    323                          #Found the key
    324                          del(data[k])
    325                      else:
    326                          #Still need to find children nodes
    327                          JsonHelper._remove_key(data[k], key, top_level=False)
    328
    329                      # Don't need to continue iterating. Node was already found
    330                      break
    331          else:
    332              raise RuntimeError("Element %s is not allowed in a JSON response" % type(data))
    333
    334          if len(key) > 0:
    335              # Not all keys were found. Can't fail here because it would make impossible testing the scenario where
    336              # a JSON payload has a key and other doesn't
    337              warn("Key '%s' does not represent a valid element" % '.'.join(orig_key))
    338
    339          if top_level:
    340              #Restore key variable
    341              # noinspection PyUnusedLocal
    342              key = list(orig_key)
    343
    344      @staticmethod
    345 -    def get_elements(data, json_path):
    346          """"
    347          Applies a JSON path to a JSON string and returns the resulting node
    348          @param data:
    349          @type data: str, dict
    350          """
    351          #If it's a string, convert to dictionary
    352          if isinstance(data, basestring):
    353              try:
    354                  json_data = json.loads(data)
    355              except ValueError:
    356                  raise RuntimeError("Value doesn't represent a valid JSON string\n%s" % item)
    357          elif type(data) is dict:
    358                  #Sort items inside lists
    359                  json_data = data
    360          else:
    361              raise BadJsonFormatError("Parameter doesn't represent a valid JSON object: %s" % item)
    362
    363          jsonpath_expr = jsonpath_rw.parse(json_path)
    364          result = []
    365          for i in jsonpath_expr.find(json_data):
    366              result.append(i.value)
    367
    368          return result
    369