Merge tag '7.0.5' into 7.0-cn-annotated

Author: Holo

00-RELEASENOTES

@@ -12,6 +12,49 @@ SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
 
+================================================================================
+Redis 7.0.5 Released Wed Sep 21 20:00:00 IST 2022
+================================================================================
+
+Upgrade urgency: SECURITY, contains fixes to security issues.
+
+Security Fixes:
+* (CVE-2022-35951) Executing a XAUTOCLAIM command on a stream key in a specific
+  state, with a specially crafted COUNT argument, may cause an integer overflow,
+  a subsequent heap overflow, and potentially lead to remote code execution.
+  The problem affects Redis versions 7.0.0 or newer
+  [reported by Xion (SeungHyun Lee) of KAIST GoN].
+
+Module API changes
+==================
+
+* Fix RM_Call execution of scripts when used with M/W/S flags to properly
+  handle script flags (#11159)
+* Fix RM_SetAbsExpire and RM_GetAbsExpire API registration (#11025, #8564)
+
+Bug Fixes
+=========
+
+* Fix a hang when eviction is combined with lazy-free and maxmemory-eviction-tenacity is set to 100 (#11237)
+* Fix a crash when a replica may attempt to set itself as its master as a result of a manual failover (#11263)
+* Fix a bug where a cluster-enabled replica node may permanently set its master's hostname to '?' (#10696)
+* Fix a crash when a Lua script returns a meta-table (#11032)
+
+Fixes for issues in previous releases of Redis 7.0
+--------------------------------------------------
+
+* Fix redis-cli to do DNS lookup before sending CLUSTER MEET (#11151)
+* Fix crash when a key is lazy expired during cluster key migration (#11176)
+* Fix AOF rewrite to fsync the old AOF file when a new one is created (#11004)
+* Fix some crashes involving a list containing entries larger than 1GB (#11242)
+* Correctly handle scripts with a non-read-only shebang on a cluster replica (#11223)
+* Fix memory leak when unloading a module (#11147)
+* Fix bug with scripts ignoring client tracking NOLOOP (#11052)
+* Fix client-side tracking breaking protocol when FLUSHDB / FLUSHALL / SWAPDB is used inside MULTI-EXEC (#11038)
+* Fix ACL: BITFIELD with GET and also SET / INCRBY can be executed with read-only key permission (#11086)
+* Fix missing sections for INFO ALL when also requesting a module info section (#11291)
+
+
 ================================================================================
 Redis 7.0.4 Released Monday Jul 18 12:00:00 IST 2022
 ================================================================================

src/aof.c

@@ -57,6 +57,7 @@ int aofFileExist(char *filename);
 int rewriteAppendOnlyFile(char *filename);
 aofManifest *aofLoadManifestFromFile(sds am_filepath);
 void aofManifestFreeAndUpdate(aofManifest *am);
+void aof_background_fsync_and_close(int fd);
 
 /* ----------------------------------------------------------------------------
  * AOF Manifest file implementation.
@@ -901,8 +902,14 @@ int openNewIncrAofForAppend(void) {
     /* If reaches here, we can safely modify the `server.aof_manifest`
      * and `server.aof_fd`. */
 
-    /* Close old aof_fd if needed. */
-    if (server.aof_fd != -1) bioCreateCloseJob(server.aof_fd);
+    /* fsync and close old aof_fd if needed. In fsync everysec it's ok to delay
+     * the fsync as long as we grantee it happens, and in fsync always the file
+     * is already synced at this point so fsync doesn't matter. */
+    if (server.aof_fd != -1) {
+        aof_background_fsync_and_close(server.aof_fd);
+        server.aof_fsync_offset = server.aof_current_size;
+        server.aof_last_fsync = server.unixtime;
+    }
     server.aof_fd = newfd;
 
     /* Reset the aof_last_incr_size. */
@@ -999,6 +1006,9 @@ int aofRewriteLimited(void) {
  * BIO thread. */
 /* bio 线程是否有 aof fysnc 任务要执行 */
 int aofFsyncInProgress(void) {
+    /* Note that we don't care about aof_background_fsync_and_close because
+     * server.aof_fd has been replaced by the new INCR AOF file fd,
+     * see openNewIncrAofForAppend. */
     return bioPendingJobsOfType(BIO_AOF_FSYNC) != 0;
 }
 
@@ -1009,6 +1019,11 @@ void aof_background_fsync(int fd) {
     bioCreateFsyncJob(fd);
 }
 
+/* Close the fd on the basis of aof_background_fsync. */
+void aof_background_fsync_and_close(int fd) {
+    bioCreateCloseJob(fd, 1);
+}
+
 /* Kills an AOFRW child process if exists */
 /* 关闭 aof rewrite 子进程 */
 void killAppendOnlyChild(void) {

src/bio.c

@@ -91,15 +91,22 @@ static unsigned long long bio_pending[BIO_NUM_OPS];
 /* This structure represents a background Job. It is only used locally to this
  * file as the API does not expose the internals at all. */
 /* 后台 IO 结构体，仅仅在本地使用，相关的 API 不会被暴露出去 */
-struct bio_job {
+typedef union bio_job {
     /* Job specific arguments.*/
     /* 后台任务持有的文件描述符 */
-    int fd; /* Fd for file based background jobs */
-    /* 懒释放函数，释放在 free_args 中存储的对象 */
-    lazy_free_fn *free_fn; /* Function that will free the provided arguments */
-    /* 要被释放的对象参数 */
-    void *free_args[]; /* List of arguments to be passed to the free function */
-};
+    struct {
+        int fd; /* Fd for file based background jobs */
+        /* 懒释放函数，释放在 free_args 中存储的对象 */
+        unsigned need_fsync:1; /* A flag to indicate that a fsync is required before
+                                * the file is closed. */
+    } fd_args;
+
+    struct {
+        lazy_free_fn *free_fn; /* Function that will free the provided arguments */
+        /* 要被释放的对象参数 */
+        void *free_args[]; /* List of arguments to be passed to the free function */
+    } free_args;
+} bio_job;
 
 void *bioProcessBackgroundJobs(void *arg);
 
@@ -150,7 +157,7 @@ void bioInit(void) {
 }
 
 /* 提交任务，将任务放入对应类型操作线程的任务列表 */
-void bioSubmitJob(int type, struct bio_job *job) {
+void bioSubmitJob(int type, bio_job *job) {
     pthread_mutex_lock(&bio_mutex[type]);
     listAddNodeTail(bio_jobs[type],job);
     bio_pending[type]++;
@@ -170,36 +177,37 @@ void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) {
     va_list valist;
     /* Allocate memory for the job structure and all required
      * arguments */
-    struct bio_job *job = zmalloc(sizeof(*job) + sizeof(void *) * (arg_count));
-    job->free_fn = free_fn;
+    bio_job *job = zmalloc(sizeof(*job) + sizeof(void *) * (arg_count));
+    job->free_args.free_fn = free_fn;
 
     va_start(valist, arg_count);
     for (int i = 0; i < arg_count; i++) {
-        job->free_args[i] = va_arg(valist, void *);
+        job->free_args.free_args[i] = va_arg(valist, void *);
     }
     va_end(valist);
     bioSubmitJob(BIO_LAZY_FREE, job);
 }
 
 /* 创建 close file 任务，填充 fd 属性，提交任务 */
-void bioCreateCloseJob(int fd) {
-    struct bio_job *job = zmalloc(sizeof(*job));
-    job->fd = fd;
+void bioCreateCloseJob(int fd, int need_fsync) {
+    bio_job *job = zmalloc(sizeof(*job));
+    job->fd_args.fd = fd;
+    job->fd_args.need_fsync = need_fsync;
 
     bioSubmitJob(BIO_CLOSE_FILE, job);
 }
 
 /* 创建 AOF file fsync 任务，填充 fd 属性，提交任务 */
 void bioCreateFsyncJob(int fd) {
-    struct bio_job *job = zmalloc(sizeof(*job));
-    job->fd = fd;
+    bio_job *job = zmalloc(sizeof(*job));
+    job->fd_args.fd = fd;
 
     bioSubmitJob(BIO_AOF_FSYNC, job);
 }
 
 /* 后台 IO 线程执行任务函数 */
 void *bioProcessBackgroundJobs(void *arg) {
-    struct bio_job *job;
+    bio_job *job;
     unsigned long type = (unsigned long) arg;
     sigset_t sigset;
 
@@ -260,14 +268,17 @@ void *bioProcessBackgroundJobs(void *arg) {
         /* Process the job accordingly to its type. */
         /* 不同的操作类型执行具体的操作 */
         if (type == BIO_CLOSE_FILE) {
+            if (job->fd_args.need_fsync) {
+                redis_fsync(job->fd_args.fd);
+            }
             /* 关闭文件 */
-            close(job->fd);
+            close(job->fd_args.fd);
         } else if (type == BIO_AOF_FSYNC) {
             /* The fd may be closed by main thread and reused for another
              * socket, pipe, or file. We just ignore these errno because
              * aof fsync did not really fail. */
             /* 做 fsync，会忽略错误，原子的设置 AOF 的 FSYNC 状态信息 */
-            if (redis_fsync(job->fd) == -1 &&
+            if (redis_fsync(job->fd_args.fd) == -1 &&
                 errno != EBADF && errno != EINVAL)
             {
                 int last_status;
@@ -283,7 +294,7 @@ void *bioProcessBackgroundJobs(void *arg) {
             }
         } else if (type == BIO_LAZY_FREE) {
             /* 调用惰性释放函数释放对象 */
-            job->free_fn(job->free_args);
+            job->free_args.free_fn(job->free_args.free_args);
         } else {
             serverPanic("Wrong job type in bioProcessBackgroundJobs().");
         }

src/bio.h

@@ -37,7 +37,7 @@ void bioInit(void);
 unsigned long long bioPendingJobsOfType(int type);
 unsigned long long bioWaitStepOfType(int type);
 void bioKillThreads(void);
-void bioCreateCloseJob(int fd);
+void bioCreateCloseJob(int fd, int need_fsync);
 void bioCreateFsyncJob(int fd);
 void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...);
 

src/cluster.c

@@ -850,10 +850,15 @@ void setClusterNodeToInboundClusterLink(clusterNode *node, clusterLink *link) {
         /* A peer may disconnect and then reconnect with us, and it's not guaranteed that
          * we would always process the disconnection of the existing inbound link before
          * accepting a new existing inbound link. Therefore, it's possible to have more than
-         * one inbound link from the same node at the same time. */
+         * one inbound link from the same node at the same time. Our cleanup logic assumes
+         * a one to one relationship between nodes and inbound links, so we need to kill
+         * one of the links. The existing link is more likely the outdated one, but it's
+         * possible the the other node may need to open another link. */
         serverLog(LL_DEBUG, "Replacing inbound link fd %d from node %.40s with fd %d",
                 node->inbound_link->conn->fd, node->name, link->conn->fd);
+        freeClusterLink(node->inbound_link);
     }
+    serverAssert(!node->inbound_link);
     node->inbound_link = link;
     link->node = node;
 }
@@ -1810,12 +1815,18 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
 /* IP -> string conversion. 'buf' is supposed to at least be 46 bytes.
  * If 'announced_ip' length is non-zero, it is used instead of extracting
  * the IP from the socket peer address. */
-void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) {
+int nodeIp2String(char *buf, clusterLink *link, char *announced_ip) {
     if (announced_ip[0] != '\0') {
         memcpy(buf,announced_ip,NET_IP_STR_LEN);
         buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */
+        return C_OK;
     } else {
-        connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL);
+        if (connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL) == C_ERR) {
+            serverLog(LL_NOTICE, "Error converting peer IP to string: %s",
+                link->conn ? connGetLastError(link->conn) : "no link");
+            return C_ERR;
+        }
+        return C_OK;
     }
 }
 
@@ -1847,7 +1858,11 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link,
      * it is safe to call during packet processing. */
     if (link == node->link) return 0;
 
-    nodeIp2String(ip,link,hdr->myip);
+    /* If the peer IP is unavailable for some reasons like invalid fd or closed
+     * link, just give up the update this time, and the update will be retried
+     * in the next round of PINGs */
+    if (nodeIp2String(ip,link,hdr->myip) == C_ERR) return 0;
+
     if (node->port == port && node->cport == cport && node->pport == pport &&
         strcmp(ip,node->ip) == 0) return 0;
 
@@ -2000,7 +2015,13 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc
         clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
                              CLUSTER_TODO_UPDATE_STATE|
                              CLUSTER_TODO_FSYNC_CONFIG);
-    } else if (myself->slaveof && myself->slaveof->slaveof) {
+    } else if (myself->slaveof && myself->slaveof->slaveof &&
+               /* In some rare case when CLUSTER FAILOVER TAKEOVER is used, it
+                * can happen that myself is a replica of a replica of myself. If
+                * this happens, we do nothing to avoid a crash and wait for the
+                * admin to repair the cluster. */
+               myself->slaveof->slaveof != myself)
+    {
         /* Safeguard against sub-replicas. A replica's master can turn itself
          * into a replica if its last slot is removed. If no other node takes
          * over the slot, there is nothing else to trigger replica migration. */
@@ -2337,7 +2358,7 @@ int clusterProcessPacket(clusterLink *link) {
              * 传入的 nodename 是 NULL，会随机生成一个节点名来赋值给 node->name */
             node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE);
             /* 发送方的端口信息我们是知道了，进行填充，缓冲区中有 */
-            nodeIp2String(node->ip,link,hdr->myip);
+            serverAssert(nodeIp2String(node->ip,link,hdr->myip) == C_OK);
             node->port = ntohs(hdr->port);
             node->pport = ntohs(hdr->pport);
             node->cport = ntohs(hdr->cport);
@@ -7074,7 +7095,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
              * slot migration, the channel will be served from the source
              * node until the migration completes with CLUSTER SETSLOT <slot>
              * NODE <node-id>. */
-            int flags = LOOKUP_NOTOUCH | LOOKUP_NOSTATS | LOOKUP_NONOTIFY;
+            int flags = LOOKUP_NOTOUCH | LOOKUP_NOSTATS | LOOKUP_NONOTIFY | LOOKUP_NOEXPIRE;
             if ((migrating_slot || importing_slot) && !is_pubsubshard)
             {
                 if (lookupKeyReadWithFlags(&server.db[0], thiskey, flags) == NULL) missing_keys++;
@@ -7088,6 +7109,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
      * without redirections or errors in all the cases. */
     if (n == NULL) return myself;
 
+    uint64_t cmd_flags = getCommandFlags(c);
     /* Cluster is globally down but we got keys? We only serve the request
      * if it is a read command and when allow_reads_when_down is enabled. */
     if (server.cluster->state != CLUSTER_OK) {
@@ -7101,7 +7123,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
              * cluster is down. */
             if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE;
             return NULL;
-        } else if (cmd->flags & CMD_WRITE) {
+        } else if (cmd_flags & CMD_WRITE) {
             /* The cluster is configured to allow read only commands */
             if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE;
             return NULL;
@@ -7139,7 +7161,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
      * involves multiple keys and we don't have them all, the only option is
      * to send a TRYAGAIN error. */
     if (importing_slot &&
-        (c->flags & CLIENT_ASKING || cmd->flags & CMD_ASKING))
+        (c->flags & CLIENT_ASKING || cmd_flags & CMD_ASKING))
     {
         if (multiple_keys && missing_keys) {
             if (error_code) *error_code = CLUSTER_REDIR_UNSTABLE;
@@ -7152,7 +7174,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
     /* Handle the read-only client case reading from a slave: if this
      * node is a slave and the request is about a hash slot our master
      * is serving, we can reply without redirection. */
-    int is_write_command = (c->cmd->flags & CMD_WRITE) ||
+    int is_write_command = (cmd_flags & CMD_WRITE) ||
                            (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE));
     if (((c->flags & CLIENT_READONLY) || is_pubsubshard) &&
         !is_write_command &&