Update type selection for string parameters (#218)

* proper cleans all builds

When invoking 'proper', the build script now invokes the build scripts
for both x86 and x64, if any of them are present.

* inc uninstallation timeout while integ-testing

wmi application used to uninstall driver by name during integration
testing can take over 1.5 minutes on slower VMs with many installed
applications.

* extend logging buffers for (CBOR) hex printers

When extended logging debugging define is enabled, the hex dumping of
binary buffers (used for CBOR replies logging) are now also dynamically
allocated and significantly larger (4k -> 4M).
This allows easier troubleshooting.

* change the ES type selection for text parameters

This commit changes the way a parameter's SQL type is resolved to an ES
type. So far, the size of the parameter was taking into account, to
select the ES type with the lowest "precision" that would accomodate it.

This worked well so far because IP type was having a precision of 0.
This precision for IP type had been updated in Elasticsearch (to maximum
IP length), which would lead to the driver tagging small length strings
as IP types.

With current change, all string type parameterss are considered as TEXT
types, leaving Elasticsearch/SQL to apply a better, context-informed
conversion.

Author: bpintea

.gitignore

@@ -8,6 +8,7 @@ installer/src/Installer/driver/
 dsneditor/EsOdbcDsnEditor/Debug/
 dsneditor/EsOdbcDsnEditorLauncher/bin/
 dsneditor/EsOdbcDsnEditorLauncher/obj/
+libs/zlib/zlib.lib
 
 # User-specific files
 *.suo

build.bat

@@ -334,11 +334,17 @@ REM CTESTS function: run CI testing
 REM PROPER function: clean up the build and libs dir.
 :PROPER
 	echo %~nx0: cleaning libs.
-	if exist %BUILD_DIR%\zlibclean.vcxproj (
-		MSBuild %BUILD_DIR%\zlibclean.vcxproj
+	if exist %BUILDS_DIR%\x86\zlibclean.vcxproj (
+		MSBuild %BUILDS_DIR%\x86\zlibclean.vcxproj
 	)
-	if exist %BUILD_DIR%\curlclean.vcxproj (
-		MSBuild %BUILD_DIR%\curlclean.vcxproj
+	if exist %BUILDS_DIR%\x64\zlibclean.vcxproj (
+		MSBuild %BUILDS_DIR%\x64\zlibclean.vcxproj
+	)
+	if exist %BUILDS_DIR%\x86\curlclean.vcxproj (
+		MSBuild %BUILDS_DIR%\x86\curlclean.vcxproj
+	)
+	if exist %BUILDS_DIR%\x64\curlclean.vcxproj (
+		MSBuild %BUILDS_DIR%\x64\curlclean.vcxproj
 	)
 	call:CLEAN
 	REM delete VisualStudio files

driver/connect.c

@@ -2581,26 +2581,36 @@ static void set_es_types(esodbc_dbc_st *dbc, SQLULEN rows_fetched,
 	esodbc_estype_st *types)
 {
 	SQLULEN i;
+	SQLINTEGER max_float_size, max_varchar_size;
 
-	assert(!dbc->max_float_size && !dbc->max_varchar_size);
+	dbc->es_types = types;
+	dbc->no_types = rows_fetched;
+
+	assert(dbc->max_float_type == NULL);
+	assert(dbc->max_varchar_type == NULL);
 
 	for (i = 0; i < rows_fetched; i ++) {
 		if (types[i].data_type == SQL_FLOAT) {
-			if (dbc->max_float_size < types[i].column_size) {
-				dbc->max_float_size = types[i].column_size;
+			max_float_size = dbc->max_float_type ?
+				dbc->max_float_type->column_size : 0;
+			if (max_float_size < types[i].column_size) {
+				dbc->max_float_type = &types[i];
 			}
 		} else if (types[i].data_type == SQL_VARCHAR) {
-			if (dbc->max_varchar_size < types[i].column_size) {
-				dbc->max_varchar_size = types[i].column_size;
+			max_varchar_size = dbc->max_varchar_type ?
+				dbc->max_varchar_type->column_size : 0;
+			if (max_varchar_size < types[i].column_size) {
+				dbc->max_varchar_type = &types[i];
 			}
 		}
 	}
 
-	DBGH(dbc, "%llu ES/SQL types available, maximum sizes supported for: "
-		"SQL_FLOAT: %ld, SQL_VARCHAR: %ld.", rows_fetched,
-		dbc->max_float_size, dbc->max_varchar_size);
-	dbc->es_types = types;
-	dbc->no_types = rows_fetched;
+	assert(dbc->max_float_type);
+	assert(dbc->max_varchar_type);
+
+	DBGH(dbc, "%lu ES/SQL types available, maximum sizes supported for: "
+		"SQL_FLOAT: %ld, SQL_VARCHAR: %ld.", (unsigned long)rows_fetched,
+		dbc->max_float_type->column_size, dbc->max_varchar_type->column_size);
 }
 
 /*

driver/defs.h

@@ -13,6 +13,9 @@
 
 /* Tracing log buffer size. */
 #define ESODBC_LOG_BUF_SIZE			(4 * 1024)
+#ifdef WITH_EXTENDED_BUFF_LOG
+#	define ESODBC_EXT_LOG_BUF_SIZE (ESODBC_LOG_BUF_SIZE * 1024)
+#endif /* WITH_EXTENDED_BUFF_LOG */
 /* Log file prefix. The format is: prefix_datime */
 #define ESODBC_LOG_FILE_PREFIX		"esodbc"
 #define ESODBC_LOG_FILE_SUFFIX		".log"

driver/handles.c

@@ -2281,7 +2281,7 @@ static BOOL consistency_check(esodbc_rec_st *rec)
 				dbc = HDRH(HDRH(desc)->stmt)->dbc;
 				if (rec->concise_type == SQL_FLOAT) {
 					assert(desc->type == DESC_TYPE_IPD);
-					column_size = dbc->max_float_size;
+					column_size = dbc->max_float_type->column_size;
 				} else {
 					if (DESC_TYPE_IS_APPLICATION(desc->type)) {
 						concise_type = sqlctype_to_es(rec->concise_type);

driver/handles.h

@@ -173,8 +173,8 @@ typedef struct struct_dbc {
 	esodbc_estype_st *es_types; /* array with ES types */
 	SQLULEN no_types; /* number of types in array */
 	/* maximum precision/length of types using same SQL data type ID */
-	SQLINTEGER max_float_size;
-	SQLINTEGER max_varchar_size;
+	esodbc_estype_st *max_varchar_type; /* pointer to TEXT type */
+	esodbc_estype_st *max_float_type; /* pointer to DOUBLE type */
 
 	CURL *curl; /* cURL handle */
 	CURLcode curl_err;

driver/log.c

@@ -32,7 +32,6 @@
 esodbc_filelog_st *_gf_log = NULL;
 
 #ifdef WITH_EXTENDED_BUFF_LOG
-#define ESODBC_EXT_LOG_BUF_SIZE (ESODBC_LOG_BUF_SIZE * 1024)
 static char **log_buffs = NULL;
 static size_t log_buffs_cnt = 0;
 static esodbc_mutex_lt log_buffs_mux = ESODBC_MUX_SINIT;
@@ -111,6 +110,7 @@ void log_cleanup()
 		free(log_buffs);
 		log_buffs = NULL;
 		log_buffs_cnt = 0;
+		ESODBC_MUX_DEL(&log_buffs_mux);
 	}
 #	endif /* WITH_EXTENDED_BUFF_LOG */
 

driver/odbc.c

@@ -57,6 +57,10 @@ static void driver_cleanup()
 {
 	connect_cleanup();
 	tinycbor_cleanup();
+
+#	ifdef WITH_EXTENDED_BUFF_LOG
+	cstr_hex_dump(NULL); /* util.[ch] */
+#	endif /* WITH_EXTENDED_BUFF_LOG */
 }
 
 BOOL WINAPI DllMain(

driver/queries.c

@@ -1305,7 +1305,10 @@ SQLRETURN copy_one_row_json(esodbc_stmt_st *stmt, SQLULEN pos)
 
 	ard = stmt->ard;
 	ird = stmt->ird;
-	rowno = stmt->tv_rows + /* current not yet counted */1;
+	rowno = stmt->tv_rows
+		+ STMT_GD_CALLING(stmt)
+		? /* SQLFetch() executed already, row counted */0
+		: /* SQLFetch() in progress, current row not yet counted */1;
 
 	with_info = FALSE;
 	/* iterate over the bound cols of one (table) row */
@@ -1496,7 +1499,10 @@ static SQLRETURN copy_one_row_cbor(esodbc_stmt_st *stmt, SQLULEN pos)
 
 	ard = stmt->ard;
 	ird = stmt->ird;
-	rowno = stmt->tv_rows + /* current row not yet counted */1;
+	rowno = stmt->tv_rows
+		+ STMT_GD_CALLING(stmt)
+		? /* SQLFetch() executed already, row counted */0
+		: /* SQLFetch() in progress, current row not yet counted */1;
 
 	with_info = FALSE;
 	/* iterate over the bound cols and contents of one (table) row */
@@ -2425,30 +2431,33 @@ SQLRETURN EsSQLPrepareW
 
 
 /* Find the ES/SQL type given in es_type; for ID matching multiple types
- * (scaled/half_float and keyword/text) use the best matching col_size, which
- * is the smallest, that's still matching (<=) the given one. This assumes the
- * types are ordered by it (as per the spec). */
+ * (scaled/half_float), but not  keyword/text, use the best matching col_size,
+ * which is the smallest, that's still matching (<=) the given one. This
+ * assumes the types are ordered by it (as per the spec). */
 esodbc_estype_st *lookup_es_type(esodbc_dbc_st *dbc,
 	SQLSMALLINT es_type, SQLULEN col_size)
 {
 	SQLULEN i;
+	SQLINTEGER sz;
 
+	/* for strings, choose text straight away: some type (IP, GEO) must coform
+	 * to a format and no content inspection is done in the driver */
+	if (es_type == SQL_VARCHAR) {
+		return dbc->max_varchar_type;
+	}
 	for (i = 0; i < dbc->no_types; i ++) {
 		if (dbc->es_types[i].data_type == es_type) {
 			if (col_size <= 0) {
 				return &dbc->es_types[i];
 			} else {
+				sz = dbc->es_types[i].column_size;
 				assert(col_size < LONG_MAX);
-				if ((SQLINTEGER)col_size <= dbc->es_types[i].column_size) {
-					return &dbc->es_types[i];
-				}
-				if (es_type == SQL_VARCHAR &&
-					dbc->es_types[i].column_size == dbc->max_varchar_size) {
+				if ((SQLINTEGER)col_size <= sz) {
 					return &dbc->es_types[i];
 				}
 				if (es_type == SQL_FLOAT &&
-					dbc->es_types[i].column_size == dbc->max_float_size) {
-					return &dbc->es_types[i];
+						sz == dbc->max_float_type->column_size) {
+					return dbc->max_float_type;
 				}
 			}
 		}
@@ -2460,51 +2469,44 @@ esodbc_estype_st *lookup_es_type(esodbc_dbc_st *dbc,
 
 /* find the matching ES/SQL type for app's SQL type, which can be an exact
  * match against ES/SQL types, but also some other valid SQL type. */
-static esodbc_estype_st *match_es_type(esodbc_rec_st *arec,
-	esodbc_rec_st *irec)
+static esodbc_estype_st *match_es_type(esodbc_rec_st *irec)
 {
-	SQLULEN i, length;
-	esodbc_dbc_st *dbc = arec->desc->hdr.stmt->hdr.dbc;
+	SQLULEN i;
+	SQLINTEGER col_sz;
+	esodbc_dbc_st *dbc = irec->desc->hdr.stmt->hdr.dbc;
 
 	for (i = 0; i < dbc->no_types; i ++) {
 		if (dbc->es_types[i].data_type == irec->concise_type) {
 			switch (irec->concise_type) {
-				/* For SQL types mappign to more than one ES/SQL type, choose
+				/* For SQL types mapping to more than one ES/SQL type, choose
 				 * the ES/SQL type with smallest "size" that covers user given
 				 * precision OR that has maximum precision (in case user's is
 				 * larger than max ES/SQL offers. */
 				case SQL_FLOAT: /* HALF_FLOAT, SCALED_FLOAT */
-					if (irec->precision <= dbc->es_types[i].column_size ||
-						dbc->es_types[i].column_size == dbc->max_float_size) {
-						return &dbc->es_types[i];
-					}
-					break;
-				case SQL_VARCHAR: /* KEYWORD, TEXT */
-					length = irec->length ? irec->length : arec->octet_length;
-					assert(length < LONG_MAX);
-					if ((SQLINTEGER)length <= dbc->es_types[i].column_size ||
-						dbc->es_types[i].column_size==dbc->max_varchar_size) {
+					col_sz = dbc->es_types[i].column_size;
+					if (irec->precision <= col_sz ||
+						col_sz == dbc->max_float_type->column_size) {
 						return &dbc->es_types[i];
 					}
 					break;
+				case SQL_VARCHAR: /* IP, CONSTANT_KEYWORD, KEYWORD, TEXT */
+					return lookup_es_type(dbc, SQL_VARCHAR, irec->precision);
 				default:
 					/* unequivocal match */
 					return &dbc->es_types[i];
 			}
 		}
 	}
 
-	/* app specified an SQL type with no direct mapping to an ES/SQL type
-	 * TODO: IP, GEO? */
+	/* app specified an SQL type with no direct mapping to an ES/SQL type */
 	switch (irec->meta_type) {
 		case METATYPE_EXACT_NUMERIC:
 			assert(irec->concise_type == SQL_DECIMAL ||
 				irec->concise_type == SQL_NUMERIC);
 			return lookup_es_type(dbc, SQL_FLOAT, irec->precision);
 
 		case METATYPE_STRING:
-			length = irec->length ? irec->length : arec->octet_length;
-			return lookup_es_type(dbc, SQL_VARCHAR, length);
+			return lookup_es_type(dbc, SQL_VARCHAR, irec->precision);
 		case METATYPE_BIN:
 			return lookup_es_type(dbc, SQL_BINARY, /*no prec*/0);
 		case METATYPE_DATE_TIME:
@@ -2698,7 +2700,7 @@ SQLRETURN EsSQLBindParameter(
 	}
 
 	arec = get_record(stmt->apd, ParameterNumber, /*grow?*/FALSE);
-	irec->es_type = match_es_type(arec, irec);
+	irec->es_type = match_es_type(irec);
 	if (! irec->es_type) {
 		/* validation shoudl have been done earlier on meta type setting
 		 * (SQL_DESC_CONCISE_TYPE) */

driver/util.c

@@ -741,13 +741,39 @@ BOOL TEST_API metadata_id_escape(wstr_st *src, wstr_st *dst, BOOL force)
  * Returns (thread local static) printed buffer, always 0-term'd. */
 char *cstr_hex_dump(const cstr_st *buff)
 {
+#	ifndef WITH_EXTENDED_BUFF_LOG
 	static thread_local char dest[ESODBC_LOG_BUF_SIZE];
+	const size_t dest_sz = sizeof(dest);
+#	else /* !WITH_EXTENDED_BUFF_LOG */
+	static char *dest = NULL;
+	static esodbc_mutex_lt dest_mux = ESODBC_MUX_SINIT;
+	const size_t dest_sz = ESODBC_EXT_LOG_BUF_SIZE;
+#	endif /* !WITH_EXTENDED_BUFF_LOG */
 	char *to, *from;
 	char *to_end, *from_end;
 	int n;
 
+
+#	ifdef WITH_EXTENDED_BUFF_LOG
+	ESODBC_MUX_LOCK(&dest_mux);
+	if (! buff) { /* free resorces */
+		if (dest) {
+			free(dest);
+			dest = NULL;
+		}
+		ESODBC_MUX_DEL(&dest_mux);
+		goto end;
+	} else if (! dest) {
+		if (! (dest = malloc(dest_sz))) {
+			ERRN("OOM for %zd bytes.", dest_sz);
+			ESODBC_MUX_UNLOCK(&dest_mux);
+			return "<OOM>";
+		}
+	}
+#	endif /* WITH_EXTENDED_BUFF_LOG */
+
 	to = dest;
-	to_end = dest + sizeof(dest);
+	to_end = dest + dest_sz;
 	from = buff->str;
 	from_end = buff->str + buff->cnt;
 	int i = 0;
@@ -761,10 +787,15 @@ char *cstr_hex_dump(const cstr_st *buff)
 	}
 	/* add the 0-terminator */
 	if (to < to_end) { /* still space for it? */
-		*to ++ = 0;
+		*to ++ = '\0';
 	} else { /* == */
-		dest[sizeof(dest) - 1] = 0; /* overwrite last position */
+		dest[dest_sz - 1] = '\0'; /* overwrite last position */
 	}
+
+#	ifdef WITH_EXTENDED_BUFF_LOG
+end:
+	ESODBC_MUX_UNLOCK(&dest_mux);
+#	endif /* WITH_EXTENDED_BUFF_LOG */
 	return dest;
 }
 

test/integration/install.py

@@ -16,8 +16,8 @@
 from elasticsearch import Elasticsearch
 
 DRIVER_BASE_NAME = "Elasticsearch ODBC Driver"
-# Uninstallation can take quite a long time (over 10s)
-INSTALLATION_TIMEOUT = 60
+# Uninstallation can take quite a long time on VMs
+INSTALLATION_TIMEOUT = 120
 
 class Installer(object):
 	_driver_path = None