core: fix resources leak in memory profiler

When memory profiler fails to start with PROFILE_ERRRUN status both
stream and ctx are not released. At the same time when memory profiler
fails to start with the PROFILE_ERRIO status both stream and ctx are
released twice. Both cases occur due to invalid return status checking.

To fix the leakage on_stop callback is called manually inside
the profiler when error on start occurs. Checks in
misc.memprof.start() are omitted.

Follows up tarantool/tarantool#5442

Reviewed-by: Igor Munkin <[email protected]>
Reviewed-by: Sergey Ostanevich <[email protected]>
Signed-off-by: Igor Munkin <[email protected]>

Author: Buristan

src/lib_misc.c

@@ -177,10 +177,6 @@ LJLIB_CF(misc_memprof_start)
   memprof_status = lj_memprof_start(L, &opt);
 
   if (LJ_UNLIKELY(memprof_status != PROFILE_SUCCESS)) {
-    if (memprof_status == PROFILE_ERRIO) {
-      fclose(ctx->stream);
-      lj_mem_free(ctx->g, ctx, sizeof(*ctx));
-    }
     switch (memprof_status) {
     case PROFILE_ERRUSE:
       lua_pushnil(L);

src/lj_memprof.c

@@ -228,8 +228,11 @@ int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt)
   lua_assert(opt->buf != NULL);
   lua_assert(opt->len != 0);
 
-  if (mp->state != MPS_IDLE)
+  if (mp->state != MPS_IDLE) {
+    /* Clean up resourses. Ignore possible errors. */
+    opt->on_stop(opt->ctx, opt->buf);
     return PROFILE_ERRRUN;
+  }
 
   /* Discard possible old errno. */
   mp->saved_errno = 0;
@@ -331,7 +334,8 @@ int lj_memprof_stop(struct lua_State *L)
 int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt)
 {
   UNUSED(L);
-  UNUSED(opt);
+  /* Clean up resourses. Ignore possible errors. */
+  opt->on_stop(opt->ctx, opt->buf);
   return PROFILE_ERRUSE;
 }