I have a nested dictionary. Is there only one way to get values out safely?
try:
example_dict['key1']['key2']
except KeyError:
pass
Or maybe python has a method like get() for nested dictionary ?
This question is related to
python
dictionary
methods
except
Yet another function for the same thing, also returns a boolean to represent whether the key was found or not and handles some unexpected errors.
'''
json : json to extract value from if exists
path : details.detail.first_name
empty path represents root
returns a tuple (boolean, object)
boolean : True if path exists, otherwise False
object : the object if path exists otherwise None
'''
def get_json_value_at_path(json, path=None, default=None):
if not bool(path):
return True, json
if type(json) is not dict :
raise ValueError(f'json={json}, path={path} not supported, json must be a dict')
if type(path) is not str and type(path) is not list:
raise ValueError(f'path format {path} not supported, path can be a list of strings like [x,y,z] or a string like x.y.z')
if type(path) is str:
path = path.strip('.').split('.')
key = path[0]
if key in json.keys():
return get_json_value_at_path(json[key], path[1:], default)
else:
return False, default
example usage:
my_json = {'details' : {'first_name' : 'holla', 'last_name' : 'holla'}}
print(get_json_value_at_path(my_json, 'details.first_name', ''))
print(get_json_value_at_path(my_json, 'details.phone', ''))
(True, 'holla')
(False, '')
There are already lots of good answers but I have come up with a function called get similar to lodash get in JavaScript land that also supports reaching into lists by index:
def get(value, keys, default_value = None):
'''
Useful for reaching into nested JSON like data
Inspired by JavaScript lodash get and Clojure get-in etc.
'''
if value is None or keys is None:
return None
path = keys.split('.') if isinstance(keys, str) else keys
result = value
def valid_index(key):
return re.match('^([1-9][0-9]*|[0-9])$', key) and int(key) >= 0
def is_dict_like(v):
return hasattr(v, '__getitem__') and hasattr(v, '__contains__')
for key in path:
if isinstance(result, list) and valid_index(key) and int(key) < len(result):
result = result[int(key)] if int(key) < len(result) else None
elif is_dict_like(result) and key in result:
result = result[key]
else:
result = default_value
break
return result
def test_get():
assert get(None, ['foo']) == None
assert get({'foo': 1}, None) == None
assert get(None, None) == None
assert get({'foo': 1}, []) == {'foo': 1}
assert get({'foo': 1}, ['foo']) == 1
assert get({'foo': 1}, ['bar']) == None
assert get({'foo': 1}, ['bar'], 'the default') == 'the default'
assert get({'foo': {'bar': 'hello'}}, ['foo', 'bar']) == 'hello'
assert get({'foo': {'bar': 'hello'}}, 'foo.bar') == 'hello'
assert get({'foo': [{'bar': 'hello'}]}, 'foo.0.bar') == 'hello'
assert get({'foo': [{'bar': 'hello'}]}, 'foo.1') == None
assert get({'foo': [{'bar': 'hello'}]}, 'foo.1.bar') == None
assert get(['foo', 'bar'], '1') == 'bar'
assert get(['foo', 'bar'], '2') == None
For nested dictionary/JSON lookups, you can use dictor
pip install dictor
dict object
{
"characters": {
"Lonestar": {
"id": 55923,
"role": "renegade",
"items": [
"space winnebago",
"leather jacket"
]
},
"Barfolomew": {
"id": 55924,
"role": "mawg",
"items": [
"peanut butter jar",
"waggy tail"
]
},
"Dark Helmet": {
"id": 99999,
"role": "Good is dumb",
"items": [
"Shwartz",
"helmet"
]
},
"Skroob": {
"id": 12345,
"role": "Spaceballs CEO",
"items": [
"luggage"
]
}
}
}
to get Lonestar's items, simply provide a dot-separated path, ie
import json
from dictor import dictor
with open('test.json') as data:
data = json.load(data)
print dictor(data, 'characters.Lonestar.items')
>> [u'space winnebago', u'leather jacket']
you can provide fallback value in case the key isnt in path
theres tons more options you can do, like ignore letter casing and using other characters besides '.' as a path separator,
Little improvement to reduce approach to make it work with list. Also using data path as string divided by dots instead of array.
def deep_get(dictionary, path):
keys = path.split('.')
return reduce(lambda d, key: d[int(key)] if isinstance(d, list) else d.get(key) if d else None, keys, dictionary)
Building up on Yoav's answer, an even safer approach:
def deep_get(dictionary, *keys):
return reduce(lambda d, key: d.get(key, None) if isinstance(d, dict) else None, keys, dictionary)
A simple class that can wrap a dict, and retrieve based on a key:
class FindKey(dict):
def get(self, path, default=None):
keys = path.split(".")
val = None
for key in keys:
if val:
if isinstance(val, list):
val = [v.get(key, default) if v else None for v in val]
else:
val = val.get(key, default)
else:
val = dict.get(self, key, default)
if not val:
break
return val
For example:
person = {'person':{'name':{'first':'John'}}}
FindDict(person).get('person.name.first') # == 'John'
If the key doesn't exist, it returns None by default. You can override that using a default= key in the FindDict wrapper -- for example`:
FindDict(person, default='').get('person.name.last') # == doesn't exist, so ''
I suggest you to try python-benedict.
It is a dict subclass that provides keypath support and much more.
Installation: pip install python-benedict
from benedict import benedict
example_dict = benedict(example_dict, keypath_separator='.')
now you can access nested values using keypath:
val = example_dict['key1.key2']
# using 'get' method to avoid a possible KeyError:
val = example_dict.get('key1.key2')
or access nested values using keys list:
val = example_dict['key1', 'key2']
# using get to avoid a possible KeyError:
val = example_dict.get(['key1', 'key2'])
It is well tested and open-source on GitHub:
https://github.com/fabiocaccamo/python-benedict
Note: I am the author of this project
You could also use python reduce:
def deep_get(dictionary, *keys):
return reduce(lambda d, key: d.get(key) if d else None, keys, dictionary)
def safeget(_dct, *_keys):
if not isinstance(_dct, dict): raise TypeError("Is not instance of dict")
def foo(dct, *keys):
if len(keys) == 0: return dct
elif not isinstance(_dct, dict): return None
else: return foo(dct.get(keys[0], None), *keys[1:])
return foo(_dct, *_keys)
assert safeget(dict()) == dict()
assert safeget(dict(), "test") == None
assert safeget(dict([["a", 1],["b", 2]]),"a", "d") == None
assert safeget(dict([["a", 1],["b", 2]]),"a") == 1
assert safeget({"a":{"b":{"c": 2}},"d":1}, "a", "b")["c"] == 2
for a second level key retrieving, you can do this:
key2_value = (example_dict.get('key1') or {}).get('key2')
Since raising an key error if one of keys is missing is a reasonable thing to do, we can even not check for it and get it as single as that:
def get_dict(d, kl):
cur = d[kl[0]]
return get_dict(cur, kl[1:]) if len(kl) > 1 else cur
Recursive method (?? ??????????)
Example dict:
foo = [{'feature_name': 'Sample Creator > Contract Details > Elements of the page',
'scenarios': [{'scenario_name': 'SC, CD, Elements of the page',
'scenario_status': 'failed',
'scenario_tags': None,
'steps': [{'duration': 0,
'name': 'I open application Stage and login by '
'SPT_LOGIN and password SPT_PWD',
'status': 'untested'},
{'duration': 0,
'name': 'I open Sample Creator query page',
'status': 'untested'},
{'duration': 7.78166389465332,
'name': 'I open application Stage and login by '
'SPT_LOGIN and password SPT_PWD',
'status': 'passed'},
{'duration': 3.985326051712036,
'name': 'I open Sample Creator query page',
'status': 'passed'},
{'duration': 2.9063704013824463,
'name': 'Enter value: '
'X-2008-CON-007,X-2011-CON-016 in '
'textarea: project_text_area sleep: 1',
'status': 'passed'},
{'duration': 4.4447715282440186,
'name': 'I press on GET DATA',
'status': 'passed'},
{'duration': 1.1209557056427002,
'name': 'Verify the top table on Contract Details',
'status': 'passed'},
{'duration': 3.8173601627349854,
'name': 'I export contract_details table by offset '
'x:100, y:150',
'status': 'passed'},
{'duration': 1.032956600189209,
'name': 'Check data of '
'sc__cd_elements_of_the_page_1 and skip '
'cols None',
'status': 'passed'},
{'duration': 0.04593634605407715,
'name': "Verify 'Number of Substances' column "
'values',
'status': 'passed'},
{'duration': 0.10199904441833496,
'name': 'Substance Sample Details bottom table '
'columns',
'status': 'passed'},
{'duration': 0.0009999275207519531,
'name': 'Verify the Substance Sample Details '
'bottom table',
'status': 'passed'},
{'duration': 3.8558616638183594,
'name': 'I export substance_sample_details table '
'by offset x:100, y:150',
'status': 'passed'},
{'duration': 1.0329277515411377,
'name': 'Check data of '
'sc__cd_elements_of_the_page_2 and skip '
'cols None',
'status': 'passed'},
{'duration': 0.2879970073699951,
'name': 'Click on AG-13369',
'status': 'passed'},
{'duration': 3.800830364227295,
'name': 'I export substance_sample_details table '
'by offset x:100, y:150',
'status': 'passed'},
{'duration': 1.0169551372528076,
'name': 'Check data of '
'sc__cd_elements_of_the_page_3 and skip '
'cols None',
'status': 'passed'},
{'duration': 1.7484464645385742,
'name': 'Select all cells, table: 2',
'status': 'passed'},
{'duration': 3.812828779220581,
'name': 'I export substance_sample_details table '
'by offset x:100, y:150',
'status': 'passed'},
{'duration': 1.0029594898223877,
'name': 'Check data of '
'sc__cd_elements_of_the_page_2 and skip '
'cols None',
'status': 'passed'},
{'duration': 1.6729373931884766,
'name': 'Set window size x:800, y:600',
'status': 'passed'},
{'duration': 30.145705699920654,
'name': 'All scrollers are placed on top 6 and far '
'left 8',
'status': 'failed'}]}]},
{'feature_name': 'Sample Creator > Substance Sample History > Elements of the '
'page',
'scenarios': [{'scenario_name': 'SC, SSH, Elements of the page',
'scenario_status': 'passed',
'scenario_tags': None,
'steps': [{'duration': 0,
'name': 'I open application Stage and login by '
'SPT_LOGIN and password SPT_PWD',
'status': 'untested'},
{'duration': 0,
'name': 'I open Sample Creator query page',
'status': 'untested'},
{'duration': 7.305850505828857,
'name': 'I open application Stage and login by '
'SPT_LOGIN and password SPT_PWD',
'status': 'passed'},
{'duration': 3.500955104827881,
'name': 'I open Sample Creator query page',
'status': 'passed'},
{'duration': 3.0419492721557617,
'name': 'Enter value: NOA401800 SYN-NOA '
'A,S4A482070C SYN-ISN-OLD '
'O,S04A482167T,S04A482190Y,CSAA796564,CSCD106701 '
'in textarea: id_text_area sleep: 1',
'status': 'passed'},
{'duration': 49.567158460617065,
'name': 'I press on GET DATA',
'status': 'passed'},
{'duration': 0.13904356956481934,
'name': 'Open substance_sample_history',
'status': 'passed'},
{'duration': 1.1039845943450928,
'name': 'Columns displayed',
'status': 'passed'},
{'duration': 3.881945848464966,
'name': 'I export export_parent_table table by '
'offset x:100, y:150',
'status': 'passed'},
{'duration': 1.0334820747375488,
'name': 'Check data of '
'sc__ssh_elements_of_the_page_1 and skip '
'cols None',
'status': 'passed'},
{'duration': 0.0319981575012207,
'name': "Title is 'Additional Details for Marked "
"Rows'",
'status': 'passed'},
{'duration': 0.08897256851196289,
'name': 'Columns displayed (the same as in top '
'table)',
'status': 'passed'},
{'duration': 25.192569971084595,
'name': 'Verify the content of the bottom table',
'status': 'passed'},
{'duration': 4.308935880661011,
'name': 'I export '
'additional_details_for_marked_rows table '
'by offset x:100, y:150',
'status': 'passed'},
{'duration': 1.0089836120605469,
'name': 'Check data of '
'sc__ssh_elements_of_the_page_1 and skip '
'cols None',
'status': 'passed'}]}]}]
Code:
def get_keys(_dict: dict, prefix: list):
prefix += list(_dict.keys())
return prefix
def _loop_elements(elems:list, prefix=None, limit=None):
prefix = prefix or []
limit = limit or 9
try:
if len(elems) != 0 and isinstance(elems, list):
for _ in elems:
if isinstance(_, dict):
get_keys(_, prefix)
for item in _.values():
_loop_elements(item, prefix, limit)
return prefix[:limit]
except TypeError:
return
>>>goo = _loop_elements(foo,limit=9)
>>>goo
['feature_name', 'scenarios', 'scenario_name', 'scenario_status', 'scenario_tags', 'steps', 'duration', 'name', 'status']
An adaptation of unutbu's answer that I found useful in my own code:
example_dict.setdefaut('key1', {}).get('key2')
It generates a dictionary entry for key1 if it does not have that key already so that you avoid the KeyError. If you want to end up a nested dictionary that includes that key pairing anyway like I did, this seems like the easiest solution.
I little changed this answer. I added checking if we're using list with numbers.
So now we can use it whichever way. deep_get(allTemp, [0], {}) or deep_get(getMinimalTemp, [0, minimalTemperatureKey], 26) etc
def deep_get(_dict, keys, default=None):
def _reducer(d, key):
if isinstance(d, dict):
return d.get(key, default)
if isinstance(d, list):
return d[key] if len(d) > 0 else default
return default
return reduce(_reducer, keys, _dict)
You can use pydash:
import pydash as _
_.get(example_dict, 'key1.key2', default='Default')
After seeing this for deeply getting attributes, I made the following to safely get nested dict values using dot notation. This works for me because my dicts are deserialized MongoDB objects, so I know the key names don't contain .s. Also, in my context, I can specify a falsy fallback value (None) that I don't have in my data, so I can avoid the try/except pattern when calling the function.
from functools import reduce # Python 3
def deepgetitem(obj, item, fallback=None):
"""Steps through an item chain to get the ultimate value.
If ultimate value or path to value does not exist, does not raise
an exception and instead returns `fallback`.
>>> d = {'snl_final': {'about': {'_icsd': {'icsd_id': 1}}}}
>>> deepgetitem(d, 'snl_final.about._icsd.icsd_id')
1
>>> deepgetitem(d, 'snl_final.about._sandbox.sbx_id')
>>>
"""
def getitem(obj, name):
try:
return obj[name]
except (KeyError, TypeError):
return fallback
return reduce(getitem, item.split('.'), obj)
A solution I've used that is similar to the double get but with the additional ability to avoid a TypeError using if else logic:
value = example_dict['key1']['key2'] if example_dict.get('key1') and example_dict['key1'].get('key2') else default_value
However, the more nested the dictionary the more cumbersome this becomes.
A recursive solution. It's not the most efficient but I find it a bit more readable than the other examples and it doesn't rely on functools.
def deep_get(d, keys):
if not keys or d is None:
return d
return deep_get(d.get(keys[0]), keys[1:])
Example
d = {'meta': {'status': 'OK', 'status_code': 200}}
deep_get(d, ['meta', 'status_code']) # => 200
deep_get(d, ['garbage', 'status_code']) # => None
A more polished version
def deep_get(d, keys, default=None):
"""
Example:
d = {'meta': {'status': 'OK', 'status_code': 200}}
deep_get(d, ['meta', 'status_code']) # => 200
deep_get(d, ['garbage', 'status_code']) # => None
deep_get(d, ['meta', 'garbage'], default='-') # => '-'
"""
assert type(keys) is list
if d is None:
return default
if not keys:
return d
return deep_get(d.get(keys[0]), keys[1:], default)
By combining all of these answer here and small changes that I made, I think this function would be useful. its safe, quick, easily maintainable.
def deep_get(dictionary, keys, default=None):
return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
Example :
>>> from functools import reduce
>>> def deep_get(dictionary, keys, default=None):
... return reduce(lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("."), dictionary)
...
>>> person = {'person':{'name':{'first':'John'}}}
>>> print (deep_get(person, "person.name.first"))
John
>>> print (deep_get(person, "person.name.lastname"))
None
>>> print (deep_get(person, "person.name.lastname", default="No lastname"))
No lastname
>>>
glom is a nice library that can into dotted queries too:
In [1]: from glom import glom
In [2]: data = {'a': {'b': {'c': 'd'}}}
In [3]: glom(data, "a.b.c")
Out[3]: 'd'
A query failure has a nice stack trace, indicating the exact failure spot:
In [4]: glom(data, "a.b.foo")
---------------------------------------------------------------------------
PathAccessError Traceback (most recent call last)
<ipython-input-4-2a3467493ac4> in <module>
----> 1 glom(data, "a.b.foo")
~/.cache/pypoetry/virtualenvs/neural-knapsack-dE7ihQtM-py3.8/lib/python3.8/site-packages/glom/core.py in glom(target, spec, **kwargs)
2179
2180 if err:
-> 2181 raise err
2182 return ret
2183
PathAccessError: error raised while processing, details below.
Target-spec trace (most recent last):
- Target: {'a': {'b': {'c': 'd'}}}
- Spec: 'a.b.foo'
glom.core.PathAccessError: could not access 'foo', part 2 of Path('a', 'b', 'foo'), got error: KeyError('foo')
Safeguard with default:
In [5]: glom(data, "a.b.foo", default="spam")
Out[5]: 'spam'
The beauty of glom is in the versatile spec parameter. For example, one can easily extract all first names from the following data:
In [8]: data = {
...: "people": [
...: {"first_name": "Alice", "last_name": "Adams"},
...: {"first_name": "Bob", "last_name": "Barker"}
...: ]
...: }
In [9]: glom(data, ("people", ["first_name"]))
Out[9]: ['Alice', 'Bob']
Read the glom docs for more examples.
While the reduce approach is neat and short, I think a simple loop is easier to grok. I've also included a default parameter.
def deep_get(_dict, keys, default=None):
for key in keys:
if isinstance(_dict, dict):
_dict = _dict.get(key, default)
else:
return default
return _dict
As an exercise to understand how the reduce one-liner worked, I did the following. But ultimately the loop approach seems more intuitive to me.
def deep_get(_dict, keys, default=None):
def _reducer(d, key):
if isinstance(d, dict):
return d.get(key, default)
return default
return reduce(_reducer, keys, _dict)
Usage
nested = {'a': {'b': {'c': 42}}}
print deep_get(nested, ['a', 'b'])
print deep_get(nested, ['a', 'b', 'z', 'z'], default='missing')
Source: Stackoverflow.com