Introduction
Dictionary, also known as symbol table, associative array or map, is an abstract data structure used to store key-value pairs.
Redis's dictionary uses a hash table as the underlying implementation. There can be multiple hash table nodes in a hash table, and each hash table node stores a key-value pair in the dictionary.
achieve
/*
* 哈希表节点
*/
typedef struct dictEntry {
// 键
void *key;
// 值
union {
void *val;
uint64_t u64;
int64_t s64;
} v;
// 指向下个哈希表节点，形成链表（解决冲突）
struct dictEntry *next;
} dictEntry;
/*
* 哈希表
*
* 每个字典都使用两个哈希表，从而实现渐进式 rehash 。
*/
typedef struct dictht {
// 哈希表数组
dictEntry **table;
// 哈希表大小
unsigned long size;
// 哈希表大小掩码，用于计算索引值
// 总是等于 size - 1
unsigned long sizemask;
// 该哈希表已有节点的数量
unsigned long used;
} dictht;
/*
* 字典类型特定函数
*/
typedef struct dictType {
// 计算哈希值的函数
unsigned int (*hashFunction)(const void *key);
// 复制键的函数
void *(*keyDup)(void *privdata, const void *key);
// 复制值的函数
void *(*valDup)(void *privdata, const void *obj);
// 对比键的函数
int (*keyCompare)(void *privdata, const void *key1, const void *key2);
// 销毁键的函数
void (*keyDestructor)(void *privdata, void *key);
// 销毁值的函数
void (*valDestructor)(void *privdata, void *obj);
} dictType;
/*
* 字典
*/
typedef struct dict {
// 类型特定函数
dictType *type;
// 私有数据（保存需要传给那些类型特定函数的可选参数）
void *privdata;
// 哈希表（一般情况下字段只会使用ht[0]哈希表，ht[1]哈希表只会在对ht[0]哈希表rehash时使用）
dictht ht[2];
// rehash 索引
// 当 rehash 不在进行时，值为 -1
int rehashidx; /* rehashing not in progress if rehashidx == -1 */
// 目前正在运行的安全迭代器的数量
int iterators; /* number of iterators currently running */
} dict;
rehash
As the operation continues, the key-value pairs stored in the hash table will gradually increase or decrease, in order to maintain the load factor of the hash table (load factor = the number of stored nodes in the hash table / the size of the hash table) within a reasonable range Within, when the hash table saves too many or too few key-value pairs, the program needs to expand or shrink the size of the hash table accordingly.
Redis performs rehash on the hash table of the dictionary as follows:
-
Allocate space for the ht[1] hash table of the dictionary. The size of this space depends on the operation to be performed:
if the expansion operation is performed, the size of ht[1] is the first one greater than or equal to ht[0].used *2 2^n;
if the shrink operation is performed, the size of ht[1] is the first 2^n greater than or equal to ht[0].used; -
Rehash all key-value pairs stored in ht[0] to ht[1]: rehash refers to recalculating the hash value and index value of the key, and then placing the key-value pair in the specified position of ht[1] on.
-
When all the key-value pairs contained in ht[0] are migrated to ht[1], release ht[0], set ht[1] to ht[0], and create a new blank hash in ht[1] Table to prepare for the next rehash.
The following is an example of rehash:
The current value of ht[0].used is 4, 4 * 2 = 8. And 8 (2^3) happens to be the first n-th power of 2 that is greater than or equal to 4, so the program will hash table ht[1] The size is set to 8.
Progressive rehash
The rehash action in Redis is not one-time, centralized completion, but multiple and progressive completion.
The purpose of this is that if the server contains many key-value pairs, and all these key-value pairs must be rehashed to ht[1] at one time, the huge amount of calculation may cause the server to stop serving for a period of time.
In order to avoid this impact, Redis uses progressive Redis :
- Allocate space for ht[1] so that the dictionary holds two hash tables ht[0] and ht[1] at the same time.
- Maintain an index counter variable rehashidx in the dictionary and set it to 0 to indicate the start of rehash work.
- During the rehash, every time the dictionary is added, deleted, searched or updated, in addition to performing the specified operation, the program will also rehash all the key-value pairs of the ht[0] hash table on the rehashidx index to ht. In [1] , when the rehash work is completed, the program adds the value of the rehashidx attribute by 1.
- As the dictionary operation continues, finally at a certain point in time, all the key-value pairs of ht[0] are rehashed to ht[1]. At this time, the rehashidx attribute is set to -1, which means that the rehash is completed.
The advantage of progressive rehash is that it adopts a divide-and-conquer approach, and amortizes the calculation work required for rehash key-value pairs to each addition, deletion, search, and update operation of the dictionary , thereby avoiding the huge calculations caused by centralized rehash the amount.
The following is an example of progressive rehash (note the change process of rehashidx):
Hash table operation during the execution of progressive rehash
Because in the process of progressive rehash, the dictionary will use two hash tables ht[0] and ht[1] at the same time, so during the progressive rehash, the dictionary is deleted, Operations such as lookups and updates are performed on two tables.
For example, the search operation will be performed on ht[0] first, and then on ht[1] if it is not found.
The key-value pairs of the add operation will all be saved in ht[1]. This measure ensures that the key-value pairs contained in ht[0] will only decrease but not increase.
Main API
// 释放给定字典节点的值
#define dictFreeVal(d, entry) \
if ((d)->type->valDestructor) \
(d)->type->valDestructor((d)->privdata, (entry)->v.val)
// 设置给定字典节点的值
#define dictSetVal(d, entry, _val_) do { \
if ((d)->type->valDup) \
entry->v.val = (d)->type->valDup((d)->privdata, _val_); \
else \
entry->v.val = (_val_); \
} while(0)
// 将一个有符号整数设为节点的值
#define dictSetSignedIntegerVal(entry, _val_) \
do { entry->v.s64 = _val_; } while(0)
// 将一个无符号整数设为节点的值
#define dictSetUnsignedIntegerVal(entry, _val_) \
do { entry->v.u64 = _val_; } while(0)
// 释放给定字典节点的键
#define dictFreeKey(d, entry) \
if ((d)->type->keyDestructor) \
(d)->type->keyDestructor((d)->privdata, (entry)->key)
// 设置给定字典节点的键
#define dictSetKey(d, entry, _key_) do { \
if ((d)->type->keyDup) \
entry->key = (d)->type->keyDup((d)->privdata, _key_); \
else \
entry->key = (_key_); \
} while(0)
// 比对两个键
#define dictCompareKeys(d, key1, key2) \
(((d)->type->keyCompare) ? \
(d)->type->keyCompare((d)->privdata, key1, key2) : \
(key1) == (key2))
// 计算给定键的哈希值
#define dictHashKey(d, key) (d)->type->hashFunction(key)
// 返回获取给定节点的键
#define dictGetKey(he) ((he)->key)
// 返回获取给定节点的值
#define dictGetVal(he) ((he)->v.val)
// 返回获取给定节点的有符号整数值
#define dictGetSignedIntegerVal(he) ((he)->v.s64)
// 返回给定节点的无符号整数值
#define dictGetUnsignedIntegerVal(he) ((he)->v.u64)
// 返回给定字典的大小
#define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size)
// 返回字典的已有节点数量
#define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used)
// 查看字典是否正在 rehash
#define dictIsRehashing(ht) ((ht)->rehashidx != -1)dictCreate: Create a dict
/*
* 重置（或初始化）给定哈希表的各项属性值
*
*
* T = O(1)
*/
static void _dictReset(dictht *ht)
{
ht->table = NULL;
ht->size = 0;
ht->sizemask = 0;
ht->used = 0;
}
/* Create a new hash table */
/*
* 创建一个新的字典
*
* T = O(1)
*/
dict *dictCreate(dictType *type,
void *privDataPtr)
{
dict *d = zmalloc(sizeof(*d));
_dictInit(d,type,privDataPtr);
return d;
}
/* Initialize the hash table */
/*
* 初始化哈希表
*
* T = O(1)
*/
int _dictInit(dict *d, dictType *type,
void *privDataPtr)
{
// 初始化两个哈希表的各项属性值
// 但暂时还不分配内存给哈希表数组
_dictReset(&d->ht[0]);
_dictReset(&d->ht[1]);
// 设置类型特定函数
d->type = type;
// 设置私有数据
d->privdata = privDataPtr;
// 设置哈希表 rehash 状态
d->rehashidx = -1;
// 设置字典的安全迭代器数量
d->iterators = 0;
return DICT_OK;
}It should be noted that when creating and initializing a dict, no space allocation size is requested for the hash table, but space is allocated when the dictEntry node is added to the dict. The initial size of the hash table is defined as follows
#define DICT_HT_INITIAL_SIZE 4dictAdd (increase): add a dictEntry node
/*
* 尝试将给定键值对添加到字典中
*
* 只有给定键 key 不存在于字典时，添加操作才会成功
*
* 添加成功返回 DICT_OK ，失败返回 DICT_ERR
*
* 最坏 T = O(N) ，平滩 O(1)
*/
int dictAdd(dict *d, void *key, void *val)
{
// 尝试添加键到字典，并返回包含了这个键的新哈希节点
// T = O(N)
dictEntry *entry = dictAddRaw(d,key);
// 键已存在，添加失败
if (!entry) return DICT_ERR;
// 键不存在，设置节点的值
// T = O(1)
dictSetVal(d, entry, val);
// 添加成功
return DICT_OK;
}
/*
* 尝试将键插入到字典中
*
* 如果键已经在字典存在，那么返回 NULL
*
* 如果键不存在，那么程序创建新的哈希节点，
* 将节点和键关联，并插入到字典，然后返回节点本身。
*
* T = O(N)
*/
dictEntry *dictAddRaw(dict *d, void *key)
{
int index;
dictEntry *entry;
dictht *ht;
// 如果条件允许的话，进行单步 rehash
// T = O(1)
if (dictIsRehashing(d)) _dictRehashStep(d);
/* Get the index of the new element, or -1 if
* the element already exists. */
// 计算键在哈希表中的索引值
// 如果值为 -1 ，那么表示键已经存在
// T = O(N)
if ((index = _dictKeyIndex(d, key)) == -1)
return NULL;
// T = O(1)
/* Allocate the memory and store the new entry */
// 如果字典正在 rehash ，那么将新键添加到 1 号哈希表
// 否则，将新键添加到 0 号哈希表
ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0];
// 为新节点分配空间
entry = zmalloc(sizeof(*entry));
// 将新节点插入到链表表头
entry->next = ht->table[index];
ht->table[index] = entry;
// 更新哈希表已使用节点数量
ht->used++;
/* Set the hash entry fields. */
// 设置新节点的键
// T = O(1)
dictSetKey(d, entry, key);
return entry;
}
/* Returns the index of a free slot that can be populated with
* a hash entry for the given 'key'.
* If the key already exists, -1 is returned.
*
* 返回可以将 key 插入到哈希表的索引位置
* 如果 key 已经存在于哈希表，那么返回 -1
*
* Note that if we are in the process of rehashing the hash table, the
* index is always returned in the context of the second (new) hash table.
*
* 注意，如果字典正在进行 rehash ，那么总是返回 1 号哈希表的索引。
* 因为在字典进行 rehash 时，新节点总是插入到 1 号哈希表。
*
* T = O(N)
*/
static int _dictKeyIndex(dict *d, const void *key)
{
unsigned int h, idx, table;
dictEntry *he;
/* Expand the hash table if needed */
// 单步 rehash
// T = O(N)
if (_dictExpandIfNeeded(d) == DICT_ERR)
return -1;
/* Compute the key hash value */
// 计算 key 的哈希值
h = dictHashKey(d, key);
// T = O(1)
for (table = 0; table <= 1; table++) {
// 计算索引值
idx = h & d->ht[table].sizemask;
/* Search if this slot does not already contain the given key */
// 查找 key 是否存在
// T = O(1)
he = d->ht[table].table[idx];
while(he) {
if (dictCompareKeys(d, key, he->key))
return -1;
he = he->next;
}
// 如果运行到这里时，说明 0 号哈希表中所有节点都不包含 key
// 如果这时 rehahs 正在进行，那么继续对 1 号哈希表进行 rehash
if (!dictIsRehashing(d)) break;
}
// 返回索引值
return idx;
}
/* Expand the hash table if needed */
/*
* 根据需要，初始化字典（的哈希表），或者对字典（的现有哈希表）进行扩展
*
* T = O(N)
*/
static int _dictExpandIfNeeded(dict *d)
{
/* Incremental rehashing already in progress. Return. */
// 渐进式 rehash 已经在进行了，直接返回
if (dictIsRehashing(d)) return DICT_OK;
/* If the hash table is empty expand it to the initial size. */
// 如果字典（的 0 号哈希表）为空，那么创建并返回初始化大小的 0 号哈希表
// T = O(1)
if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE);
/* If we reached the 1:1 ratio, and we are allowed to resize the hash
* table (global setting) or we should avoid it but the ratio between
* elements/buckets is over the "safe" threshold, we resize doubling
* the number of buckets. */
// 一下两个条件之一为真时，对字典进行扩展
// 1）字典已使用节点数和字典大小之间的比率接近 1：1
// 并且 dict_can_resize 为真
// 2）已使用节点数和字典大小之间的比率超过 dict_force_resize_ratio
if (d->ht[0].used >= d->ht[0].size &&
(dict_can_resize ||
d->ht[0].used/d->ht[0].size > dict_force_resize_ratio))
{
// 新哈希表的大小至少是目前已使用节点数的两倍
// T = O(N)
return dictExpand(d, d->ht[0].used*2);
}
return DICT_OK;
}
/* Expand or create the hash table */
/*
* 创建一个新的哈希表，并根据字典的情况，选择以下其中一个动作来进行：
*
* 1) 如果字典的 0 号哈希表为空，那么将新哈希表设置为 0 号哈希表
* 2) 如果字典的 0 号哈希表非空，那么将新哈希表设置为 1 号哈希表，
* 并打开字典的 rehash 标识，使得程序可以开始对字典进行 rehash
*
* size 参数不够大，或者 rehash 已经在进行时，返回 DICT_ERR 。
*
* 成功创建 0 号哈希表，或者 1 号哈希表时，返回 DICT_OK 。
*
* T = O(N)
*/
int dictExpand(dict *d, unsigned long size)
{
// 新哈希表
dictht n; /* the new hash table */
// 根据 size 参数，计算哈希表的大小
// T = O(1)
unsigned long realsize = _dictNextPower(size);
/* the size is invalid if it is smaller than the number of
* elements already inside the hash table */
// 不能在字典正在 rehash 时进行
// size 的值也不能小于 0 号哈希表的当前已使用节点
if (dictIsRehashing(d) || d->ht[0].used > size)
return DICT_ERR;
/* Allocate the new hash table and initialize all pointers to NULL */
// 为哈希表分配空间，并将所有指针指向 NULL
n.size = realsize;
n.sizemask = realsize-1;
// T = O(N)
n.table = zcalloc(realsize*sizeof(dictEntry*));
n.used = 0;
/* Is this the first initialization? If so it's not really a rehashing
* we just set the first hash table so that it can accept keys. */
// 如果 0 号哈希表为空，那么这是一次初始化：
// 程序将新哈希表赋给 0 号哈希表的指针，然后字典就可以开始处理键值对了。
if (d->ht[0].table == NULL) {
d->ht[0] = n;
return DICT_OK;
}
/* Prepare a second hash table for incremental rehashing */
// 如果 0 号哈希表非空，那么这是一次 rehash ：
// 程序将新哈希表设置为 1 号哈希表，
// 并将字典的 rehash 标识打开，让程序可以开始对字典进行 rehash
d->ht[1] = n;
d->rehashidx = 0;
return DICT_OK;
/* 顺带一提，上面的代码可以重构成以下形式：
if (d->ht[0].table == NULL) {
// 初始化
d->ht[0] = n;
} else {
// rehash
d->ht[1] = n;
d->rehashidx = 0;
}
return DICT_OK;
*/
}
/* Our hash table capability is a power of two */
/*
* 计算第一个大于等于 size 的 2 的 N 次方，用作哈希表的值
*
* T = O(1)
*/
static unsigned long _dictNextPower(unsigned long size)
{
unsigned long i = DICT_HT_INITIAL_SIZE;
if (size >= LONG_MAX) return LONG_MAX;
while(1) {
if (i >= size)
return i;
i *= 2;
}
}
dictDelete (delete): delete a dictEntry node
/*
* 从字典中删除包含给定键的节点
*
* 并且调用键值的释放函数来删除键值
*
* 找到并成功删除返回 DICT_OK ，没找到则返回 DICT_ERR
* T = O(1)
*/
int dictDelete(dict *ht, const void *key) {
return dictGenericDelete(ht,key,0);
}
/* Search and remove an element */
/*
* 查找并删除包含给定键的节点
*
* 参数 nofree 决定是否调用键和值的释放函数
* 0 表示调用，1 表示不调用
*
* 找到并成功删除返回 DICT_OK ，没找到则返回 DICT_ERR
*
* T = O(1)
*/
static int dictGenericDelete(dict *d, const void *key, int nofree)
{
unsigned int h, idx;
dictEntry *he, *prevHe;
int table;
// 字典（的哈希表）为空
if (d->ht[0].size == 0) return DICT_ERR; /* d->ht[0].table is NULL */
// 进行单步 rehash ，T = O(1)
if (dictIsRehashing(d)) _dictRehashStep(d);
// 计算哈希值
h = dictHashKey(d, key);
// 遍历哈希表
// T = O(1)
for (table = 0; table <= 1; table++) {
// 计算索引值
idx = h & d->ht[table].sizemask;
// 指向该索引上的链表
he = d->ht[table].table[idx];
prevHe = NULL;
// 遍历链表上的所有节点
// T = O(1)
while(he) {
if (dictCompareKeys(d, key, he->key)) {
// 超找目标节点
/* Unlink the element from the list */
// 从链表中删除
if (prevHe)
prevHe->next = he->next;
else
d->ht[table].table[idx] = he->next;
// 释放调用键和值的释放函数？
if (!nofree) {
dictFreeKey(d, he);
dictFreeVal(d, he);
}
// 释放节点本身
zfree(he);
// 更新已使用节点数量
d->ht[table].used--;
// 返回已找到信号
return DICT_OK;
}
prevHe = he;
he = he->next;
}
// 如果执行到这里，说明在 0 号哈希表中找不到给定键
// 那么根据字典是否正在进行 rehash ，决定要不要查找 1 号哈希表
if (!dictIsRehashing(d)) break;
}
// 没找到
return DICT_ERR; /* not found */
}
/* Clear & Release the hash table */
/*
* 删除并释放整个字典
*
* T = O(N)
*/
void dictRelease(dict *d)
{
// 删除并清空两个哈希表
_dictClear(d,&d->ht[0],NULL);
_dictClear(d,&d->ht[1],NULL);
// 释放节点结构
zfree(d);
}
/* Destroy an entire dictionary */
/*
* 删除哈希表上的所有节点，并重置哈希表的各项属性
*
* T = O(N)
*/
int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
unsigned long i;
/* Free all the elements */
// 遍历整个哈希表
// T = O(N)
for (i = 0; i < ht->size && ht->used > 0; i++) {
dictEntry *he, *nextHe;
if (callback && (i & 65535) == 0) callback(d->privdata);
// 跳过空索引
if ((he = ht->table[i]) == NULL) continue;
// 遍历整个链表
// T = O(1)
while(he) {
nextHe = he->next;
// 删除键
dictFreeKey(d, he);
// 删除值
dictFreeVal(d, he);
// 释放节点
zfree(he);
// 更新已使用节点计数
ht->used--;
// 处理下个节点
he = nextHe;
}
}
/* Free the table and the allocated cache structure */
// 释放哈希表结构
zfree(ht->table);
/* Re-initialize the table */
// 重置哈希表属性
_dictReset(ht);
return DICT_OK; /* never fails */
}
dictFind (check): Find the dictEntry where the key is stored in the current dict according to the key
/*
* 返回字典中包含键 key 的节点
*
* 找到返回节点，找不到返回 NULL
*
* T = O(1)
*/
dictEntry *dictFind(dict *d, const void *key)
{
dictEntry *he;
unsigned int h, idx, table;
// 字典（的哈希表）为空
if (d->ht[0].size == 0) return NULL; /* We don't have a table at all */
// 如果条件允许的话，进行单步 rehash
if (dictIsRehashing(d)) _dictRehashStep(d);
// 计算键的哈希值
h = dictHashKey(d, key);
// 在字典的哈希表中查找这个键
// T = O(1)
for (table = 0; table <= 1; table++) {
// 计算索引值
idx = h & d->ht[table].sizemask;
// 遍历给定索引上的链表的所有节点，查找 key
he = d->ht[table].table[idx];
// T = O(1)
while(he) {
if (dictCompareKeys(d, key, he->key))
return he;
he = he->next;
}
// 如果程序遍历完 0 号哈希表，仍然没找到指定的键的节点
// 那么程序会检查字典是否在进行 rehash ，
// 然后才决定是直接返回 NULL ，还是继续查找 1 号哈希表
if (!dictIsRehashing(d)) return NULL;
}
// 进行到这里时，说明两个哈希表都没找到
return NULL;
}
dictReplace (modified): modify the member v (value) of a dictEntry node
/* Add an element, discarding the old if the key already exists.
*
* 将给定的键值对添加到字典中，如果键已经存在，那么删除旧有的键值对。
*
* Return 1 if the key was added from scratch, 0 if there was already an
* element with such key and dictReplace() just performed a value update
* operation.
*
* 如果键值对为全新添加，那么返回 1 。
* 如果键值对是通过对原有的键值对更新得来的，那么返回 0 。
*
* T = O(N)
*/
int dictReplace(dict *d, void *key, void *val)
{
dictEntry *entry, auxentry;
/* Try to add the element. If the key
* does not exists dictAdd will suceed. */
// 尝试直接将键值对添加到字典
// 如果键 key 不存在的话，添加会成功
// T = O(N)
if (dictAdd(d, key, val) == DICT_OK)
return 1;
/* It already exists, get the entry */
// 运行到这里，说明键 key 已经存在，那么找出包含这个 key 的节点
// T = O(1)
entry = dictFind(d, key);
/* Set the new value and free the old one. Note that it is important
* to do that in this order, as the value may just be exactly the same
* as the previous one. In this context, think to reference counting,
* you want to increment (set), and then decrement (free), and not the
* reverse. */
// 先保存原有的值的指针
auxentry = *entry;
// 然后设置新的值
// T = O(1)
dictSetVal(d, entry, val);
// 然后释放旧值
// T = O(1)
dictFreeVal(d, &auxentry);
return 0;
}
/* dictReplaceRaw() is simply a version of dictAddRaw() that always
* returns the hash entry of the specified key, even if the key already
* exists and can't be added (in that case the entry of the already
* existing key is returned.)
*
* See dictAddRaw() for more information. */
/*
* dictAddRaw() 根据给定 key 释放存在，执行以下动作：
*
* 1) key 已经存在，返回包含该 key 的字典节点
* 2) key 不存在，那么将 key 添加到字典
*
* 不论发生以上的哪一种情况，
* dictAddRaw() 都总是返回包含给定 key 的字典节点。
*
* T = O(N)
*/
dictEntry *dictReplaceRaw(dict *d, void *key) {
// 使用 key 在字典中查找节点
// T = O(1)
dictEntry *entry = dictFind(d,key);
// 如果节点找到了直接返回节点，否则添加并返回一个新节点
// T = O(N)
return entry ? entry : dictAddRaw(d,key);
}rehash
When the dictEntry node of the dictionary expands to a certain scale, the linked list of hash conflicts will become longer and longer, so that the original O(1) hash operation is converted to the O(n) linked list traversal, which leads to the addition, deletion, modification and query of the dictionary (CRUD). ) Is becoming less efficient. At this time, redis will use dictht[1] in the dict structure to expand a new hash table (call dictExpand), and then slowly migrate the nodes of dictht[0] to dictht[1], and use dictht [1] Replace dictht[0] (call dictRehash).
The specific expansion process is as follows:
1. Call dictAdd to add a dictEntry node to the dict.
2. Call _dictKeyIndex to find which index should be placed in the hash table. By the way, call _dictExpandIfNeeded to determine whether to expand dictht[1].
3. If the conditions are met: the number of dictEntry nodes of dictht[0]/the number of indexes of the hash table>=1, then dictExpand is called; if the number of dictEntry nodes/the index of the hash table>=dict_force_resize_ratio (default is 5), then the dictExpand expansion is forced dictht[1].
It should be noted that the above condition is not the condition of rehash, but the condition of expanding dictht[1] and turning on the dict rehash switch. (Because rehash is not completed at once. If there are a lot of dictEntry nodes, the rehash process will take a long time to block other operations. So redis uses progressive rehash, which is carried out step by step. The rehash method will be introduced in detail below). What dictExpand does is just expand the hash table of dictht[1] to an appropriate size, and then set the rehashidx of dict from -1 to 0 (turn on the rehash switch). After the rehash switch is turned on, the dict will perform a rehash every time the dict is added, deleted, modified and checked (CRUD), and the linked list in an index in the hash table of dictht[0] is moved to dictht[1]. The condition of rehash is whether rehashidx is -1.
rehash way
- Active way: call dictRehashMilliseconds for one millisecond. Call it in the serverCron of redis. You can tell by the name that it is a timed event prepared for the redis server. dictRehash of 1ms is executed each time, which is simple and rude. .
- Passive method: When CRUD calls dictAdd, dicFind, dictDelete, dictGetRandomKey and other functions, it will call _dictRehashStep to migrate a non-empty node in the hash table.
/* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
/*
* 在给定毫秒数内，以 100 步为单位，对字典进行 rehash 。
*
* T = O(N)
*/
int dictRehashMilliseconds(dict *d, int ms) {
// 记录开始时间
long long start = timeInMilliseconds();
int rehashes = 0;
while(dictRehash(d,100)) {
rehashes += 100;
// 如果时间已过，跳出
if (timeInMilliseconds()-start > ms) break;
}
return rehashes;
}
/* This function performs just a step of rehashing, and only if there are
* no safe iterators bound to our hash table. When we have iterators in the
* middle of a rehashing we can't mess with the two hash tables otherwise
* some element can be missed or duplicated.
*
* 在字典不存在安全迭代器的情况下，对字典进行单步 rehash 。
*
* 字典有安全迭代器的情况下不能进行 rehash ，
* 因为两种不同的迭代和修改操作可能会弄乱字典。
*
* This function is called by common lookup or update operations in the
* dictionary so that the hash table automatically migrates from H1 to H2
* while it is actively used.
*
* 这个函数被多个通用的查找、更新操作调用，
* 它可以让字典在被使用的同时进行 rehash 。
*
* T = O(1)
*/
static void _dictRehashStep(dict *d) {
if (d->iterators == 0) dictRehash(d,1);
}
/* This function performs just a step of rehashing, and only if there are
* no safe iterators bound to our hash table. When we have iterators in the
* middle of a rehashing we can't mess with the two hash tables otherwise
* some element can be missed or duplicated.
*
* 在字典不存在安全迭代器的情况下，对字典进行单步 rehash 。
*
* 字典有安全迭代器的情况下不能进行 rehash ，
* 因为两种不同的迭代和修改操作可能会弄乱字典。
*
* This function is called by common lookup or update operations in the
* dictionary so that the hash table automatically migrates from H1 to H2
* while it is actively used.
*
* 这个函数被多个通用的查找、更新操作调用，
* 它可以让字典在被使用的同时进行 rehash 。
*
* T = O(1)
*/
static void _dictRehashStep(dict *d) {
if (d->iterators == 0) dictRehash(d,1);
}
/* Performs N steps of incremental rehashing. Returns 1 if there are still
* keys to move from the old to the new hash table, otherwise 0 is returned.
*
* 执行 N 步渐进式 rehash 。
*
* 返回 1 表示仍有键需要从 0 号哈希表移动到 1 号哈希表，
* 返回 0 则表示所有键都已经迁移完毕。
*
* Note that a rehashing step consists in moving a bucket (that may have more
* than one key as we use chaining) from the old to the new hash table.
*
* 注意，每步 rehash 都是以一个哈希表索引（桶）作为单位的，
* 一个桶里可能会有多个节点，
* 被 rehash 的桶里的所有节点都会被移动到新哈希表。
*
* T = O(N)
*/
int dictRehash(dict *d, int n) {
// 只可以在 rehash 进行中时执行
if (!dictIsRehashing(d)) return 0;
// 进行 N 步迁移
// T = O(N)
while(n--) {
dictEntry *de, *nextde;
/* Check if we already rehashed the whole table... */
// 如果 0 号哈希表为空，那么表示 rehash 执行完毕
// T = O(1)
if (d->ht[0].used == 0) {
// 释放 0 号哈希表
zfree(d->ht[0].table);
// 将原来的 1 号哈希表设置为新的 0 号哈希表
d->ht[0] = d->ht[1];
// 重置旧的 1 号哈希表
_dictReset(&d->ht[1]);
// 关闭 rehash 标识
d->rehashidx = -1;
// 返回 0 ，向调用者表示 rehash 已经完成
return 0;
}
/* Note that rehashidx can't overflow as we are sure there are more
* elements because ht[0].used != 0 */
// 确保 rehashidx 没有越界
assert(d->ht[0].size > (unsigned)d->rehashidx);
// 略过数组中为空的索引，找到下一个非空索引
while(d->ht[0].table[d->rehashidx] == NULL) d->rehashidx++;
// 指向该索引的链表表头节点
de = d->ht[0].table[d->rehashidx];
/* Move all the keys in this bucket from the old to the new hash HT */
// 将链表中的所有节点迁移到新哈希表
// T = O(1)
while(de) {
unsigned int h;
// 保存下个节点的指针
nextde = de->next;
/* Get the index in the new hash table */
// 计算新哈希表的哈希值，以及节点插入的索引位置
h = dictHashKey(d, de->key) & d->ht[1].sizemask;
// 插入节点到新哈希表
de->next = d->ht[1].table[h];
d->ht[1].table[h] = de;
// 更新计数器
d->ht[0].used--;
d->ht[1].used++;
// 继续处理下个节点
de = nextde;
}
// 将刚迁移完的哈希表索引的指针设为空
d->ht[0].table[d->rehashidx] = NULL;
// 更新 rehash 索引
d->rehashidx++;
}
return 1;
}Dictionary iterator
/*
* 字典迭代器
*
* 如果 safe 属性的值为 1 ，那么在迭代进行的过程中，
* 程序仍然可以执行 dictAdd 、 dictFind 和其他函数，对字典进行修改。
*
* 如果 safe 不为 1 ，那么程序只会调用 dictNext 对字典进行迭代，
* 而不对字典进行修改。
*/
typedef struct dictIterator {
// 被迭代的字典
dict *d;
// table ：正在被迭代的哈希表号码，值可以是 0 或 1 。
// index ：迭代器当前所指向的哈希表索引位置。
// safe ：标识这个迭代器是否安全
int table, index, safe;
// entry ：当前迭代到的节点的指针
// nextEntry ：当前迭代节点的下一个节点
// 因为在安全迭代器运作时， entry 所指向的节点可能会被修改，
// 所以需要一个额外的指针来保存下一节点的位置，
// 从而防止指针丢失
dictEntry *entry, *nextEntry;
long long fingerprint; /* 指纹是一个64位的数字，它代表了给定时间内字典的状态 */
} dictIterator;
/*
* 创建并返回给定字典的不安全迭代器
*
* T = O(1)
*/
dictIterator *dictGetIterator(dict *d)
{
dictIterator *iter = zmalloc(sizeof(*iter));
iter->d = d;
iter->table = 0;
iter->index = -1;
iter->safe = 0;
iter->entry = NULL;
iter->nextEntry = NULL;
return iter;
}
/*
* 创建并返回给定节点的安全迭代器
*
* T = O(1)
*/
dictIterator *dictGetSafeIterator(dict *d) {
dictIterator *i = dictGetIterator(d);
// 设置安全迭代器标识
i->safe = 1;
return i;
}
/*
* 返回迭代器指向的当前节点
*
* 字典迭代完毕时，返回 NULL
*
* T = O(1)
*/
dictEntry *dictNext(dictIterator *iter)
{
while (1) {
// 进入这个循环有两种可能：
// 1) 这是迭代器第一次运行
// 2) 当前索引链表中的节点已经迭代完（NULL 为链表的表尾）
if (iter->entry == NULL) {
// 指向被迭代的哈希表
dictht *ht = &iter->d->ht[iter->table];
// 初次迭代时执行
if (iter->index == -1 && iter->table == 0) {
// 如果是安全迭代器，那么更新安全迭代器计数器
if (iter->safe)
iter->d->iterators++;
// 如果是不安全迭代器，那么计算指纹
else
iter->fingerprint = dictFingerprint(iter->d);
}
// 更新索引
iter->index++;
// 如果迭代器的当前索引大于当前被迭代的哈希表的大小
// 那么说明这个哈希表已经迭代完毕
if (iter->index >= (signed) ht->size) {
// 如果正在 rehash 的话，那么说明 1 号哈希表也正在使用中
// 那么继续对 1 号哈希表进行迭代
if (dictIsRehashing(iter->d) && iter->table == 0) {
iter->table++;
iter->index = 0;
ht = &iter->d->ht[1];
// 如果没有 rehash ，那么说明迭代已经完成
} else {
break;
}
}
// 如果进行到这里，说明这个哈希表并未迭代完
// 更新节点指针，指向下个索引链表的表头节点
iter->entry = ht->table[iter->index];
} else {
// 执行到这里，说明程序正在迭代某个链表
// 将节点指针指向链表的下个节点
iter->entry = iter->nextEntry;
}
// 如果当前节点不为空，那么也记录下该节点的下个节点
// 因为安全迭代器有可能会将迭代器返回的当前节点删除
if (iter->entry) {
/* We need to save the 'next' here, the iterator user
* may delete the entry we are returning. */
iter->nextEntry = iter->entry->next;
return iter->entry;
}
}
// 迭代完毕
return NULL;
}
/*
* 释放给定字典迭代器
*
* T = O(1)
*/
void dictReleaseIterator(dictIterator *iter)
{
if (!(iter->index == -1 && iter->table == 0)) {
// 释放安全迭代器时，安全迭代器计数器减一
if (iter->safe)
iter->d->iterators--;
// 释放不安全迭代器时，验证指纹是否有变化
else
assert(iter->fingerprint == dictFingerprint(iter->d));
}
zfree(iter);
}hash algorithm
/* 对正整数计算hash */
unsigned int dictIntHashFunction(unsigned int key)
{
key += ~(key << 15);
key ^= (key >> 10);
key += (key << 3);
key ^= (key >> 6);
key += ~(key << 11);
key ^= (key >> 16);
return key;
}
/*对字符串进行hash*/
static uint32_t dict_hash_function_seed = 5381;
unsigned int dictGenHashFunction(const void *key, int len) {
/* 'm' and 'r' are mixing constants generated offline.
They're not really 'magic', they just happen to work well. */
uint32_t seed = dict_hash_function_seed;
const uint32_t m = 0x5bd1e995;
const int r = 24;
/* Initialize the hash to a 'random' value */
uint32_t h = seed ^ len;
/* Mix 4 bytes at a time into the hash */
const unsigned char *data = (const unsigned char *)key;
while(len >= 4) {
uint32_t k = *(uint32_t*)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
/* Handle the last few bytes of the input array */
switch(len) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0]; h *= m;
};
/* Do a few final mixes of the hash to ensure the last few
* bytes are well-incorporated. */
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return (unsigned int)h;
}
/* 对字符串进行hash，不区分大小写 */
unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
unsigned int hash = (unsigned int)dict_hash_function_seed;
while (len--)
hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */
return hash;
}