Fix: support for CBOR chunked strings (#236)

* add support for chunked strings

ES will split the large string values in chunks (of ~4KiB), even though
the entire set of chunks is contained within the answer (and thus
needless). This commit:
- introduces a new CBOR utility function that will fail to get string
references, if the value is chunked; this is mostly useful for fetching
object key names, which are part of the protocol and short;
- will now allow the cursor string to be chunked, in which case it'll be
allocated by the tinycbor library.

* cbor: support row string values received chunked

This commit adds support for reading row string values when these are
spread over multiple chunks.

Author: bpintea

CMakeLists.txt

@@ -342,6 +342,7 @@ list(APPEND DRV_SRC ${CMAKE_BINARY_DIR}/cborparser.c)
 list(APPEND DRV_SRC ${TINYCBOR_PATH_SRC}/src/cborvalidation.c)
 list(APPEND DRV_SRC ${TINYCBOR_PATH_SRC}/src/cborerrorstrings.c)
 list(APPEND DRV_SRC ${TINYCBOR_PATH_SRC}/src/cborencoder.c)
+list(APPEND DRV_SRC ${TINYCBOR_PATH_SRC}/src/cborparser_dup_string.c)
 list(APPEND DRV_SRC
 	${TINYCBOR_PATH_SRC}/src/cborencoder_close_container_checked.c)
 set(TINYCBOR_INC ${TINYCBOR_PATH_SRC}/src)

driver/connect.c

@@ -1626,7 +1626,8 @@ static BOOL parse_es_version_cbor(esodbc_dbc_st *dbc, cstr_st *rsp_body,
 	}
 
 	/* fetch `version` value */
-	res = cbor_value_get_string_chunk(&iter_ver, &version->str, &version->cnt);
+	res = cbor_value_get_unchunked_string(&iter_ver, &version->str,
+			&version->cnt);
 	CHK_RES("failed to fetch " ESINFO_KEY_NUMBER " value");
 
 	/* Note: containers must be "left" (cbor_value_leave_container()) if ever

driver/handles.h

@@ -314,7 +314,8 @@ typedef struct struct_desc {
 	assert(DESC_TYPE_IS_IMPLEMENTATION(_rec->desc->type) && _rec->es_type)
 
 struct resultset_cbor {
-	cstr_st curs; /* ES'es cursor; refs req's body */
+	cstr_st curs; /* ES'es cursor */
+	BOOL curs_allocd; /* curs.str is allocated (and reassembled) */
 	CborValue rows_obj; /* top object rows container (EsSQLRowCount()) */
 	CborValue rows_iter; /* iterator over received rows; refs req's body */
 	wstr_st cols_buff /* columns descriptions; refs allocated chunk */;

driver/queries.c

@@ -179,6 +179,17 @@ void clear_resultset(esodbc_stmt_st *stmt, BOOL on_close)
 		} else {
 			assert(! stmt->rset.pack.cbor.cols_buff.str);
 		}
+		if (stmt->rset.pack.cbor.curs_allocd) {
+			free(stmt->rset.pack.cbor.curs.str);
+			stmt->rset.pack.cbor.curs.str = NULL;
+			stmt->rset.pack.cbor.curs_allocd = false;
+		} else if (stmt->rset.pack.cbor.curs.str) {
+			/* the cursor is contained entirely in the received body */
+			assert(stmt->rset.body.str < stmt->rset.pack.cbor.curs.str); // &&
+			assert(stmt->rset.pack.cbor.curs.str +
+					stmt->rset.pack.cbor.curs.cnt <
+					stmt->rset.body.str + stmt->rset.body.cnt);
+		}
 	}
 	memset(&stmt->rset, 0, sizeof(stmt->rset));
 
@@ -521,7 +532,7 @@ static BOOL iterate_on_columns(esodbc_stmt_st *stmt, CborValue columns)
 			ERRH(stmt, "invalid non-text element '" PACK_PARAM_COL_NAME "'.");
 			return FALSE;
 		}
-		res = cbor_value_get_string_chunk(&name_obj, &name_cstr.str,
+		res = cbor_value_get_unchunked_string(&name_obj, &name_cstr.str,
 				&name_cstr.cnt);
 		CHK_RES(stmt, "can't fetch value of '" PACK_PARAM_COL_NAME "' elem");
 
@@ -553,7 +564,7 @@ static BOOL iterate_on_columns(esodbc_stmt_st *stmt, CborValue columns)
 			ERRH(stmt, "invalid non-text element '" PACK_PARAM_COL_TYPE "'.");
 			return FALSE;
 		}
-		res = cbor_value_get_string_chunk(&type_obj, &type_cstr.str,
+		res = cbor_value_get_unchunked_string(&type_obj, &type_cstr.str,
 				&type_cstr.cnt);
 		CHK_RES(stmt, "can't fetch value of '" PACK_PARAM_COL_TYPE "' elem");
 		/* U8MB_TO_U16WC fails with 0-len source */
@@ -727,9 +738,20 @@ static SQLRETURN attach_answer_cbor(esodbc_stmt_st *stmt)
 		}
 		/* should have been cleared by now */
 		assert(! stmt->rset.pack.cbor.curs.cnt);
-		res = cbor_value_get_string_chunk(&curs_obj,
+		res = cbor_value_get_unchunked_string(&curs_obj,
 				&stmt->rset.pack.cbor.curs.str,
 				&stmt->rset.pack.cbor.curs.cnt);
+		if (res == CborErrorUnknownLength) {
+			assert(stmt->rset.pack.cbor.curs_allocd == false);
+			/* cursor is in chunked string; get it assembled in one chunk */
+			res = cbor_value_dup_text_string(&curs_obj,
+					&stmt->rset.pack.cbor.curs.str,
+					&stmt->rset.pack.cbor.curs.cnt,
+					&curs_obj);
+			if (res == CborNoError) {
+				stmt->rset.pack.cbor.curs_allocd = true;
+			}
+		}
 		CHK_RES(stmt, "failed to read '" PACK_PARAM_CURSOR "' value");
 		DBGH(stmt, "new paginating cursor: [%zd] `" LCPDL "`.",
 			stmt->rset.pack.cbor.curs.cnt, LWSTR(&stmt->rset.pack.cbor.curs));

driver/tinycbor.c

@@ -12,9 +12,9 @@
  * Note: these can't be freed per thread, since
  * DllMain(DLL_THREAD_ATTACH/DLL_THREAD_DETACH) is optional (and apps are
  * inconsistent even calling attach-detach for same thread). */
-static wchar_t **u16buffs = NULL;
-static size_t u16buff_cnt = 0;
-static esodbc_mutex_lt u16buff_mux = ESODBC_MUX_SINIT;
+static void **utf_buffs = NULL;
+static size_t utf_buff_cnt = 0;
+static esodbc_mutex_lt utf_buff_mux = ESODBC_MUX_SINIT;
 
 /* advance an iterator of an "entered" JSON-sytle map to the value for the
  * given key, if that exists */
@@ -189,31 +189,30 @@ CborError cbor_container_is_empty(CborValue cont, BOOL *empty)
 	return CborNoError;
 }
 
-static BOOL enlist_utf16_buffer(wchar_t *old, wchar_t *new)
+static BOOL enlist_utf_buffer(void *old, void *new)
 {
-	wchar_t **r;
+	void **r;
 	size_t i;
 
-	if (! old) {
-		/* new entry must be inserted into list */
-		ESODBC_MUX_LOCK(&u16buff_mux);
-		r = realloc(u16buffs, (u16buff_cnt + 1) * sizeof(wchar_t *));
+	if (! old) { /* new entry must be inserted into list */
+		ESODBC_MUX_LOCK(&utf_buff_mux);
+		r = realloc(utf_buffs, (utf_buff_cnt + 1) * sizeof(void *));
 		if (r) {
-			u16buffs = r;
-			u16buffs[u16buff_cnt ++] = new;
+			utf_buffs = r;
+			utf_buffs[utf_buff_cnt ++] = new;
 		}
-		ESODBC_MUX_UNLOCK(&u16buff_mux);
-	} else {
-		ESODBC_MUX_LOCK(&u16buff_mux);
+		ESODBC_MUX_UNLOCK(&utf_buff_mux);
+	} else { /* old entry has be reallocated, store its updated ref */
+		ESODBC_MUX_LOCK(&utf_buff_mux);
 		r = NULL;
-		for (i = 0; i < u16buff_cnt; i ++) {
-			if (u16buffs[i] == old) {
-				r = &u16buffs[i];
-				u16buffs[i] = new;
+		for (i = 0; i < utf_buff_cnt; i ++) {
+			if (utf_buffs[i] == old) {
+				r = &utf_buffs[i];
+				utf_buffs[i] = new;
 				break;
 			}
 		}
-		ESODBC_MUX_UNLOCK(&u16buff_mux);
+		ESODBC_MUX_UNLOCK(&utf_buff_mux);
 	}
 
 	return !!r;
@@ -222,31 +221,82 @@ static BOOL enlist_utf16_buffer(wchar_t *old, wchar_t *new)
 void tinycbor_cleanup()
 {
 	size_t i;
-	for (i = 0; i < u16buff_cnt; i ++) {
-		free(u16buffs[i]);
+	for (i = 0; i < utf_buff_cnt; i ++) {
+		free(utf_buffs[i]);
 	}
 	if (i) {
-		free(u16buffs);
+		free(utf_buffs);
 	}
 }
 
+static BOOL enlarge_buffer(void *str_ptr, size_t new_cnt, size_t usize)
+{
+	wstr_st r; /* reallocated */
+	wstr_st *wbuff = (wstr_st *)str_ptr;
+	/* the two string struct types should remain identical (ptr, size_t),
+	 * for the above cast to work also for cstr_st inputs */
+	assert(sizeof(cstr_st) == sizeof(wstr_st));
+	assert((void *)wbuff->str == (void *)((cstr_st *)str_ptr)->str);
+	assert(wbuff->cnt == ((cstr_st *)str_ptr)->cnt);
+
+	/* double scratchpad size until exceeding min needed space.
+	 * condition on equality, to allow for a 0-term */
+	for (r.cnt = (0 < wbuff->cnt && wbuff->cnt < (size_t)-1) ? wbuff->cnt :
+			ESODBC_BODY_BUF_START_SIZE; r.cnt <= new_cnt; r.cnt *= 2) {
+		;
+	}
+	if (! (r.str = realloc(wbuff->str, r.cnt * usize))) {
+		return false;
+	}
+	if (! enlist_utf_buffer(wbuff->str, r.str)) {
+		/* it should only possibly fail on 1st allocation per-thread (since
+		 * the rest of invocations are to swap the pointers) */
+		assert(! wbuff->str);
+		free(r.str);
+		return false;
+	} else {
+		*wbuff = r;
+	}
+	DBG("new UTF conv. buffer @0x%p, size %zu, usize: %zu.", wbuff->str,
+		wbuff->cnt, usize);
+	return true;
+}
+
 /* Fetches and converts a(n always UTF8) text string to UTF16 wide char.
  * Uses a dynamically allocated thread-local buffer.
  * 0-terminates the string */
 CborError cbor_value_get_utf16_wstr(CborValue *it, wstr_st *utf16)
 {
+	/* .cnt needs to be non-zero, for U8MB_TO_U16WC() to fail on 1st invoc. */
 	static thread_local wstr_st wbuff = {.str = NULL, .cnt = (size_t)-1};
-	wstr_st r; /* reallocated */
+	static thread_local cstr_st cbuff = {0};
 	cstr_st mb_str; /* multibyte string */
 	CborError res;
 	int n;
+	size_t len;
 
 	assert(cbor_value_is_text_string(it));
 	/* get the multibyte string to convert */
-	res = cbor_value_get_string_chunk(it, &mb_str.str, &mb_str.cnt);
-	if (res != CborNoError) {
-		return res;
+	if (cbor_value_is_length_known(it)) { /* str contained in single chunk? */
+		res = cbor_value_get_string_chunk(it, &mb_str.str, &mb_str.cnt);
+		if (res != CborNoError) {
+			return res;
+		}
+	} else { /* string is spread across multiple chunks */
+		res = cbor_value_calculate_string_length(it, &len);
+		if (res != CborNoError) {
+			return res;
+		}
+		if (cbuff.cnt <= len && !enlarge_buffer(&cbuff, len, sizeof(char))) {
+			return CborErrorOutOfMemory;
+		}
+		mb_str = cbuff;
+		res = cbor_value_copy_text_string(it, mb_str.str, &mb_str.cnt, it);
+		if (res != CborNoError) {
+			return res;
+		}
 	}
+
 	/* attempt string conversion */
 	while ((n = U8MB_TO_U16WC(mb_str.str, mb_str.cnt, wbuff.str,
 					wbuff.cnt)) <= 0) {
@@ -264,7 +314,6 @@ CborError cbor_value_get_utf16_wstr(CborValue *it, wstr_st *utf16)
 			} /* else: buffer hasn't yet been allocated or is too small */
 			/* what's the minimum space needed? */
 			if ((n = U8MB_TO_U16WC(mb_str.str, mb_str.cnt, NULL, 0)) < 0) {
-				TRACE;
 				return CborErrorInvalidUtf8TextString;
 			}
 		} else {
@@ -273,28 +322,13 @@ CborError cbor_value_get_utf16_wstr(CborValue *it, wstr_st *utf16)
 				break;
 			}
 		}
-		/* double scratchpad size until exceeding min needed space.
-		 * condition on equality, to allow for a 0-term */
-		for (r.cnt = wbuff.cnt < (size_t)-1 ? wbuff.cnt :
-				ESODBC_BODY_BUF_START_SIZE; r.cnt <= (size_t)n; r.cnt *= 2) {
-			;
-		}
-		if (! (r.str = realloc(wbuff.str, r.cnt))) {
+		if (! enlarge_buffer(&wbuff, (size_t)n, sizeof(wchar_t))) {
 			return CborErrorOutOfMemory;
 		}
-		if (! enlist_utf16_buffer(wbuff.str, r.str)) {
-			/* it should only fail on 1st allocation per-thread */
-			assert(! wbuff.str);
-			free(r.str);
-			return CborErrorOutOfMemory;
-		} else {
-			wbuff = r;
-		}
-		DBG("new UTF8/16 conv. buffer @0x%p, size %zu.", wbuff.str, wbuff.cnt);
 	}
 
 	/* U8MB_TO_U16WC() will only convert the 0-term if counted in input*/
-	wbuff.str[n] = '\0'; /* set, but not counted */
+	wbuff.str[n] = L'\0'; /* set, but not counted */
 	utf16->str = wbuff.str;
 	utf16->cnt = n;
 	return CborNoError;

driver/tinycbor.h

@@ -95,4 +95,15 @@ void tinycbor_cleanup();
 CborError cbor_value_get_string_chunk(CborValue *it,
 	const char **bufferptr, size_t *len);
 
+static inline CborError cbor_value_get_unchunked_string(CborValue *it,
+	const char **bufferptr, size_t *len)
+{
+	return cbor_value_is_length_known(it)
+		/* string is all in one chunk */
+		? cbor_value_get_string_chunk(it, bufferptr, len)
+		/* if the string is chunked, fail the call */
+		: CborErrorUnknownLength;
+}
+
+
 #endif /* __TINYCBOR_H__ */