Testing: Use CTest fixtures to manipulate files

 - Fixes jacobwilliams#413
 - Use fixtures to delete old json outputs before re-running tests, then copy
   pristine inputs back into the build directory

Author: zbeekman

CMakeLists.txt

@@ -289,13 +289,11 @@ if ( ENABLE_TESTS )
 
   find_program ( JSONLINT jsonlint )
   find_program ( DIFF     diff )
-  file ( COPY "${CMAKE_SOURCE_DIR}/files"
-    DESTINATION "${CMAKE_BINARY_DIR}/" )
 
-  set ( DATA_DIR "${CMAKE_BINARY_DIR}/files" )
+  set ( DATA_DIR "${CMAKE_SOURCE_DIR}/files" )
 
   set_directory_properties ( PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES
-    "${DATA_DIR}/test2.json;${DATA_DIR}/test4.json;${FORD_CLEAN_OUTPUTS}" )
+    "${FORD_CLEAN_OUTPUTS}" )
 
   # Validate input
   if ( JSONLINT )
@@ -309,26 +307,38 @@ if ( ENABLE_TESTS )
     foreach ( VALID_JSON ${JSON_INPUTS} )
       get_filename_component ( TESTNAME "${VALID_JSON}" NAME )
       add_test ( NAME validate-${TESTNAME}
-	WORKING_DIRECTORY "${DATA_DIR}/inputs"
-	COMMAND ${JSONLINT} "--allow=nonescape-characters" "${VALID_JSON}" )
+	      WORKING_DIRECTORY "${DATA_DIR}/inputs"
+	      COMMAND ${JSONLINT} "--allow=nonescape-characters" "${VALID_JSON}" )
     endforeach ()
 
     foreach ( INVALID ${INVALID_JSON} )
       get_filename_component ( TESTNAME "${INVALID}" NAME )
       add_test ( NAME validate-${TESTNAME}
-	WORKING_DIRECTORY "${DATA_DIR}/inputs"
-	COMMAND ${JSONLINT} "${INVALID}" )
+	      WORKING_DIRECTORY "${DATA_DIR}/inputs"
+	      COMMAND ${JSONLINT} "${INVALID}" )
       set_property ( TEST validate-${TESTNAME}
-	PROPERTY
-	WILL_FAIL TRUE)
+	      PROPERTY
+	      WILL_FAIL TRUE)
     endforeach ()
   endif ()
 
+  add_test(NAME jf-cleanup-fixture
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_BINARY_DIR}/files")
+  set_tests_properties(jf-cleanup-fixture
+    PROPERTIES FIXTURES_SETUP JF)
+  add_test(NAME jf-setup-fixture
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND ${CMAKE_COMMAND} -E copy_directory "${DATA_DIR}" "${CMAKE_BINARY_DIR}/files")
+  set_tests_properties(jf-setup-fixture
+    PROPERTIES FIXTURES_SETUP JF
+    DEPENDS jf-cleanup-fixture)
+
   set ( UNIT_TESTS '' )
   foreach ( UNIT_TEST ${JF_TEST_SRCS} )
     get_filename_component ( TEST ${UNIT_TEST} NAME_WE )
     if(MSVC_IDE)
-    link_directories(${CMAKE_BINARY_DIR}/lib)
+      link_directories(${CMAKE_BINARY_DIR}/lib)
     endif()
     add_executable ( ${TEST} EXCLUDE_FROM_ALL ${UNIT_TEST} )
     target_link_libraries ( ${TEST} ${LIB_NAME} )
@@ -340,11 +350,13 @@ if ( ENABLE_TESTS )
     add_test( NAME ${TEST}
       WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin
       COMMAND ./${TEST})
+    set_tests_properties( ${TEST}
+      PROPERTIES FIXTURES_REQUIRED JF)
     list ( APPEND UNIT_TESTS ${TEST} )
     if ( JSONLINT )
       set_property ( TEST ${TEST}
-	APPEND
-	PROPERTY DEPENDS validate-input1 validate-input2 )
+	      APPEND
+	      PROPERTY DEPENDS validate-input1 validate-input2 )
     endif()
   endforeach ( UNIT_TEST )
 
@@ -353,34 +365,37 @@ if ( ENABLE_TESTS )
     PROPERTY DEPENDS jf_test_02 )
 
   # Validate output
+  file( GLOB EXPECTED_OUTPUTS "${DATA_DIR}/expected-outputs/*.json")
+  if (NOT ${ENABLE_UNICODE})
+    list( REMOVE_ITEM EXPECTED_OUTPUTS "${DATA_DIR}/expected-outputs/hello-world-ucs4.json")
+  endif()
+  list( REMOVE_ITEM EXPECTED_OUTPUTS "${DATA_DIR}/expected-outputs/example2.json")
+
   if ( JSONLINT )
-    file ( GLOB JSON_FILES "${DATA_DIR}/*.json" )
-    foreach ( JSON_FILE ${JSON_FILES} )
+    foreach ( JSON_FILE ${EXPECTED_OUTPUTS} )
       get_filename_component ( TESTNAME ${JSON_FILE} NAME )
       add_test ( NAME validate-output-${TESTNAME}
-	WORKING_DIRECTORY "${DATA_DIR}"
-	COMMAND ${JSONLINT} "--allow=nonescape-characters" ${TESTNAME} )
+	      WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/files"
+	      COMMAND ${JSONLINT} "--allow=nonescape-characters" ${TESTNAME} )
       set_property ( TEST validate-output-${TESTNAME}
-	APPEND
-	PROPERTY
-	DEPENDS ${UNIT_TESTS}
-	REQUIRED_FILES ${JSON_FILES} )
+	      APPEND
+	      PROPERTY
+	      DEPENDS ${UNIT_TESTS})
     endforeach ( JSON_FILE )
   endif ()
 
   # Check output for differences
   if ( DIFF )
-    file ( GLOB JSON_FILES "${DATA_DIR}/*.json" )
-    foreach ( JSON_FILE ${JSON_FILES} )
-      get_filename_component ( JSON_STEM ${JSON_FILE} NAME_WE )
-      add_test ( NAME regression-${JSON_STEM}.json
-	WORKING_DIRECTORY "${DATA_DIR}"
-	COMMAND ${DIFF} -q ${JSON_STEM}.json expected-outputs/${JSON_STEM}.json )
-      set_property ( TEST regression-${JSON_STEM}.json
-	APPEND
-	PROPERTY
-	DEPENDS ${UNIT_TESTS}
-	REQUIRED_FILES ${JSON_FILES} )
+    foreach ( JSON_FILE ${EXPECTED_OUTPUTS} )
+      get_filename_component (OUTPUT ${JSON_FILE} NAME )
+      add_test ( NAME regression-${OUTPUT}
+	      WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/files"
+	      COMMAND ${DIFF} -q ${OUTPUT} expected-outputs/${OUTPUT} )
+      set_property ( TEST regression-${OUTPUT}
+	      APPEND
+	      PROPERTY
+	      DEPENDS ${UNIT_TESTS}
+	      REQUIRED_FILES ${EXPECTED_OUTPUTS} )
     endforeach ( JSON_FILE )
   else ()
     message ( WARNING

files/expected-outputs/test12.json

@@ -85,7 +85,7 @@
     "only one value"
   ],
   "page": [
-    "The quick brown fox",
+    "The quick brown fox     ",
     "jumps over the lazy dog."
   ]
 }

files/expected-outputs/test2.json

@@ -1,7 +1,7 @@
 {
   "inputs": {
-    "t0": 0.1E+0,
-    "tf": 0.11E+1,
+    "t0": 0.10000000000000001E+0,
+    "tf": 0.11000000000000001E+1,
     "x0": 0.9999E+4,
     "integer_scalar": 1,
     "integer_array": [
@@ -20,7 +20,14 @@
       false,
       true
     ],
-    "null_variable": null
+    "null_variable": null,
+    "special chars": "\\ /",
+    "special chars in key \\ /": "\\ /",
+    "bspace": "\b",
+    "horizontal_tab": "\t",
+    "newline": "\n",
+    "formfeed": "\f",
+    "carriage_return": "\r"
   },
   "trajectory": [
     {
@@ -58,35 +65,35 @@
     },
     {
       "VARIABLE": "Vx",
-      "UNITS": "km\/s",
+      "UNITS": "km/s",
       "FRAME": "J2000",
       "CENTER": "EARTH",
       "DATA": [
         0.1E-2,
         0.2E-2,
-        0.3E-2
+        0.30000000000000001E-2
       ]
     },
     {
       "VARIABLE": "Vy",
-      "UNITS": "km\/s",
+      "UNITS": "km/s",
       "FRAME": "J2000",
       "CENTER": "EARTH",
       "DATA": [
         0.2E-2,
         0.2E-1,
-        0.3E-2
+        0.30000000000000001E-2
       ]
     },
     {
       "VARIABLE": "Vz",
-      "UNITS": "km\/s",
+      "UNITS": "km/s",
       "FRAME": "J2000",
       "CENTER": "EARTH",
       "DATA": [
-        0.3E-2,
-        0.3E-1,
-        0.4E-1
+        0.30000000000000001E-2,
+        0.29999999999999999E-1,
+        0.40000000000000001E-1
       ]
     }
   ]

files/expected-outputs/test21.json

@@ -0,0 +1,13 @@
+{
+  "value": [
+    0.14142135623730951E+1,
+    0.17320508075688772E+1,
+    0.26457513110645907E+1,
+    0.14142135623730951E+3,
+    0.54772255750516615E+3,
+    0.26457513110645905E+4,
+    0.17976931348623157E+309,
+    0.22250738585072014E-307,
+    0.22204460492503131E-15
+  ]
+}