How to merge dicts, collecting values from matching keys?

Here's a general solution that will handle an arbitrary amount of dictionaries, with cases when keys are in only some of the dictionaries:

from collections import defaultdict


d1 = {1: 2, 3: 4}
d2 = {1: 6, 3: 7}


dd = defaultdict(list)


for d in (d1, d2): # you can list as many input dicts as you want here
for key, value in d.items():
dd[key].append(value)
    

print(dd) # result: defaultdict(<type 'list'>, {1: [2, 6], 3: [4, 7]})

def merge(d1, d2, merge):
result = dict(d1)
for k,v in d2.iteritems():
if k in result:
result[k] = merge(result[k], v)
else:
result[k] = v
return result


d1 = {'a': 1, 'b': 2}
d2 = {'a': 1, 'b': 3, 'c': 2}
print merge(d1, d2, lambda x, y:(x,y))


{'a': (1, 1), 'c': 2, 'b': (2, 3)}

assuming all keys are always present in all dicts:

ds = [d1, d2]
d = {}
for k in d1.iterkeys():
d[k] = tuple(d[k] for d in ds)

Note: In Python 3.x use below code:

ds = [d1, d2]
d = {}
for k in d1.keys():
d[k] = tuple(d[k] for d in ds)

and if the dic contain numpy arrays:

ds = [d1, d2]
d = {}
for k in d1.keys():
d[k] = np.concatenate(list(d[k] for d in ds))

Here is one approach you can use which would work even if both dictonaries don't have same keys:

d1 = {'a':'test','b':'btest','d':'dreg'}
d2 = {'a':'cool','b':'main','c':'clear'}


d = {}


for key in set(d1.keys() + d2.keys()):
try:
d.setdefault(key,[]).append(d1[key])
except KeyError:
pass


try:
d.setdefault(key,[]).append(d2[key])
except KeyError:
pass


print d

This would generate below input:

{'a': ['test', 'cool'], 'c': ['clear'], 'b': ['btest', 'main'], 'd': ['dreg']}

If you only have d1 and d2,

from collections import defaultdict


d = defaultdict(list)
for a, b in d1.items() + d2.items():
d[a].append(b)

A compact possibility

d1={'a':1,'b':2}
d2={'c':3,'d':4}
context={**d1, **d2}
context
{'b': 2, 'c': 3, 'd': 4, 'a': 1}

dict1 = {'m': 2, 'n': 4}
dict2 = {'n': 3, 'm': 1}

Making sure that the keys are in the same order:

dict2_sorted = {i:dict2[i] for i in dict1.keys()}


keys = dict1.keys()
values = zip(dict1.values(), dict2_sorted.values())
dictionary = dict(zip(keys, values))

gives:

{'m': (2, 1), 'n': (4, 3)}

To supplement the two-list solutions, here is a solution for processing a single list.

A sample list (NetworkX-related; manually formatted here for readability):

ec_num_list = [((src, tgt), ec_num['ec_num']) for src, tgt, ec_num in G.edges(data=True)]


print('\nec_num_list:\n{}'.format(ec_num_list))
ec_num_list:
[((82, 433), '1.1.1.1'),
((82, 433), '1.1.1.2'),
((22, 182), '1.1.1.27'),
((22, 3785), '1.2.4.1'),
((22, 36), '6.4.1.1'),
((145, 36), '1.1.1.37'),
((36, 154), '2.3.3.1'),
((36, 154), '2.3.3.8'),
((36, 72), '4.1.1.32'),
...]

Note the duplicate values for the same edges (defined by the tuples). To collate those "values" to their corresponding "keys":

from collections import defaultdict
ec_num_collection = defaultdict(list)
for k, v in ec_num_list:
ec_num_collection[k].append(v)


print('\nec_num_collection:\n{}'.format(ec_num_collection.items()))
ec_num_collection:
[((82, 433), ['1.1.1.1', '1.1.1.2']),   ## << grouped "values"
((22, 182), ['1.1.1.27']),
((22, 3785), ['1.2.4.1']),
((22, 36), ['6.4.1.1']),
((145, 36), ['1.1.1.37']),
((36, 154), ['2.3.3.1', '2.3.3.8']),    ## << grouped "values"
((36, 72), ['4.1.1.32']),
...]

If needed, convert that list to dict:

ec_num_collection_dict = {k:v for k, v in zip(ec_num_collection, ec_num_collection)}


print('\nec_num_collection_dict:\n{}'.format(dict(ec_num_collection)))
ec_num_collection_dict:
{(82, 433): ['1.1.1.1', '1.1.1.2'],
(22, 182): ['1.1.1.27'],
(22, 3785): ['1.2.4.1'],
(22, 36): ['6.4.1.1'],
(145, 36): ['1.1.1.37'],
(36, 154): ['2.3.3.1', '2.3.3.8'],
(36, 72): ['4.1.1.32'],
...}

References

• [this thread] How to merge multiple dicts with same key?
• [Python docs] https://docs.python.org/3.7/library/collections.html#collections.defaultdict

From blubb answer:

You can also directly form the tuple using values from each list

ds = [d1, d2]
d = {}
for k in d1.keys():
d[k] = (d1[k], d2[k])

This might be useful if you had a specific ordering for your tuples

ds = [d1, d2, d3, d4]
d = {}
for k in d1.keys():
d[k] = (d3[k], d1[k], d4[k], d2[k]) #if you wanted tuple in order of d3, d1, d4, d2

This function merges two dicts even if the keys in the two dictionaries are different:

def combine_dict(d1, d2):
return {
k: tuple(d[k] for d in (d1, d2) if k in d)
for k in set(d1.keys()) | set(d2.keys())
}

Example:

d1 = {
'a': 1,
'b': 2,
}
d2` = {
'b': 'boat',
'c': 'car',
}
combine_dict(d1, d2)
# Returns: {
#    'a': (1,),
#    'b': (2, 'boat'),
#    'c': ('car',)
# }

Assume that you have the list of ALL keys (you can get this list by iterating through all dictionaries and get their keys). Let's name it listKeys. Also:

• listValues is the list of ALL values for a single key that you want to merge.
• allDicts: all dictionaries that you want to merge.

result = {}
for k in listKeys:
listValues = [] #we will convert it to tuple later, if you want.
for d in allDicts:
try:
fileList.append(d[k]) #try to append more values to a single key
except:
pass
if listValues: #if it is not empty
result[k] = typle(listValues) #convert to tuple, add to new dictionary with key k

This library helped me, I had a dict list of nested keys with the same name but with different values, every other solution kept overriding those nested keys.

https://pypi.org/project/deepmerge/

from deepmerge import always_merger


def process_parms(args):
temp_list = []
for x in args:
with open(x, 'r') as stream:
temp_list.append(yaml.safe_load(stream))


return always_merger.merge(*temp_list)

If keys are nested:

d1 = { 'key1': { 'nkey1': 'x1' }, 'key2': { 'nkey2': 'y1' } }
d2 = { 'key1': { 'nkey1': 'x2' }, 'key2': { 'nkey2': 'y2' } }

ds = [d1, d2]
d = {}
for k in d1.keys():
for k2 in d1[k].keys():
d.setdefault(k, {})
d[k].setdefault(k2, [])
d[k][k2] = tuple(d[k][k2] for d in ds)

yields:

{'key1': {'nkey1': ('x1', 'x2')}, 'key2': {'nkey2': ('y1', 'y2')}}

Assuming there are two dictionaries with exact same keys, below is the most succinct way of doing it (python3 should be used for both the solution).

d1 = {'a': 1, 'b': 2, 'c':3}
d2 = {'a': 5, 'b': 6, 'c':7}


# get keys from one of the dictionary
ks = [k for k in d1.keys()]


print(ks)
['a', 'b', 'c']


# call values from each dictionary on available keys
d_merged = {k: (d1[k], d2[k]) for k in ks}


print(d_merged)
{'a': (1, 5), 'b': (2, 6), 'c': (3, 7)}


# to merge values as list
d_merged = {k: [d1[k], d2[k]] for k in ks}
print(d_merged)
{'a': [1, 5], 'b': [2, 6], 'c': [3, 7]}

If there are two dictionaries with some common keys, but a few different keys, a list of all the keys should be prepared.

d1 = {'a': 1, 'b': 2, 'c':3, 'd': 9}
d2 = {'a': 5, 'b': 6, 'c':7, 'e': 4}


# get keys from one of the dictionary
d1_ks = [k for k in d1.keys()]
d2_ks = [k for k in d2.keys()]


all_ks = set(d1_ks + d2_ks)


print(all_ks)
['a', 'b', 'c', 'd', 'e']


# call values from each dictionary on available keys
d_merged = {k: [d1.get(k), d2.get(k)] for k in all_ks}


print(d_merged)
{'d': [9, None], 'a': [1, 5], 'b': [2, 6], 'c': [3, 7], 'e': [None, 4]}

A better representation for two or more dicts with the same keys is a pandas Data Frame IMO:

d1 = {"key1": "x1", "key2": "y1"}
d2 = {"key1": "x2", "key2": "y2"}
d3 = {"key1": "x3", "key2": "y3"}


d1_df = pd.DataFrame.from_dict(d1, orient='index')
d2_df = pd.DataFrame.from_dict(d2, orient='index')
d3_df = pd.DataFrame.from_dict(d3, orient='index')


fin_df = pd.concat([d1_df, d2_df, d3_df], axis=1).T.reset_index(drop=True)
fin_df


key1 key2
0   x1   y1
1   x2   y2
2   x3   y3

Using below method we can merge two dictionaries having same keys.

def update_dict(dict1: dict, dict2: dict) -> dict:
output_dict = {}
for key in dict1.keys():
output_dict.update({key: []})
if type(dict1[key]) != str:
for value in dict1[key]:
output_dict[key].append(value)
else:
output_dict[key].append(dict1[key])
if type(dict2[key]) != str:
for value in dict2[key]:
output_dict[key].append(value)
else:
output_dict[key].append(dict2[key])


return output_dict

Input: d1 = {key1: x1, key2: y1} d2 = {key1: x2, key2: y2}
Output: {'key1': ['x1', 'x2'], 'key2': ['y1', 'y2']}

There is a great library funcy doing what you need in a just one, short line.

from funcy import join_with
from pprint import pprint


d1 = {"key1": "x1", "key2": "y1"}
d2 = {"key1": "x2", "key2": "y2"}


list_of_dicts = [d1, d2]


merged_dict = join_with(tuple, list_of_dicts)


pprint(merged_dict)

Output:

{'key1': ('x1', 'x2'), 'key2': ('y1', 'y2')}

More info here: funcy -> join_with.

Modifying this answer to create a dictionary of tuples (what the OP asked for), instead of a dictionary of lists:

from collections import defaultdict


d1 = {1: 2, 3: 4}
d2 = {1: 6, 3: 7}


dd = defaultdict(tuple)


for d in (d1, d2): # you can list as many input dicts as you want here
for key, value in d.items():
dd[key] += (value,)


print(dd)

The above prints the following:

defaultdict(<class 'tuple'>, {1: (2, 6), 3: (4, 7)})

dicts = [dict1,dict2,dict3]
out   = dict(zip(dicts[0].keys(),[[dic[list(dic.keys())[key]] for dic in dicts] for key in range(0,len(dicts[0]))]))

d1 = {'A': 'a', 'B': 'b'}
d2 = {'A': 'c', 'B': 'd'}
d3 = {'A': 'e', 'B': 'f'}


dnew = {
    

k : [d1.get(k),d2.get(k),d3.get(k)] for k in d1.keys() | d2.keys() | d3.keys()
    

}


print(dnew)


"""
{'A': ['a', 'c', 'e'], 'B': ['b', 'd', 'f']}


"""