Patch leveldb for Windows UTF16 path support (#1315)

jonsimantov · web-flow · commit c5214c0be452 · 2023-05-12T09:43:51.000-07:00
* Use patched Firestore SDK

* Also patch leveldb if building RTDB without Firestore.

* Add release note for the fix.

* Update leveldb version number patch.

* Updated release note.
diff --git a/cmake/external/firestore.patch.txt b/cmake/external/firestore.patch.txt
@@ -14,12 +14,3 @@ index 920bf2928..c5c9cc7ee 100644
  
  ExternalProject_Get_property(snappy SOURCE_DIR)
  set(snappy_source_dir "${SOURCE_DIR}")
-@@ -39,7 +42,7 @@ ExternalProject_Add(
-   DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR}
-   DOWNLOAD_NAME leveldb-${version}.tar.gz
-   URL https://github.com/google/leveldb/archive/${version}.tar.gz
--  URL_HASH SHA256=55423cac9e3306f4a9502c738a001e4a339d1a38ffbee7572d4a07d5d63949b2
-+  URL_HASH SHA256=9a37f8a6174f09bd622bc723b55881dc541cd50747cbd08831c2a82d620f6d76
- 
-   PREFIX ${PROJECT_BINARY_DIR}
- 
diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake
@@ -18,6 +18,9 @@ if(TARGET leveldb)
   return()
 endif()
 
+set(patch_file 
+  ${CMAKE_CURRENT_LIST_DIR}/../../scripts/git/patches/leveldb/0001-leveldb-1.23-windows-paths.patch)
+
 # This version must be kept in sync with the version in firestore.patch.txt.
 # If this version ever changes then make sure to update the version in
 # firestore.patch.txt accordingly.
@@ -27,9 +30,8 @@ ExternalProject_Add(
   leveldb
 
   DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR}
-  DOWNLOAD_NAME leveldb-${version}.tar.gz
-  URL https://github.com/google/leveldb/archive/${version}.tar.gz
-  URL_HASH SHA256=9a37f8a6174f09bd622bc723b55881dc541cd50747cbd08831c2a82d620f6d76
+  GIT_REPOSITORY https://github.com/google/leveldb.git
+  GIT_TAG "${version}"
 
   PREFIX ${PROJECT_BINARY_DIR}
 
@@ -38,4 +40,5 @@ ExternalProject_Add(
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
   HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
+  PATCH_COMMAND git apply ${patch_file} && git gc --aggressive
 )
diff --git a/release_build_files/readme.md b/release_build_files/readme.md
@@ -627,6 +627,13 @@ workflow use only during the development of your app, not for publicly shipping
 code.
 
 ## Release Notes
+### Upcoming Release
+-   Changes
+    - Database/Firestore (Desktop): Fixed a crash on Windows when the user's
+      home directory contains non-ANSI characters (Unicode above U+00FF).
+    - Storage (Desktop): Fixed a crash on Windows when uploading files from a
+      path containing non-ANSI characters (Unicode above U+00FF).
+
 ### 11.0.1
 -   Changes
     - Auth (iOS): Fixed a crash in `Credential::is_valid()` when an `AuthResult`
diff --git a/scripts/git/patches/leveldb/0001-leveldb-1.23-windows-paths.patch b/scripts/git/patches/leveldb/0001-leveldb-1.23-windows-paths.patch
@@ -0,0 +1,307 @@
+diff --git a/util/env_windows.cc b/util/env_windows.cc
+index 449f564..812c728 100644
+--- a/util/env_windows.cc
++++ b/util/env_windows.cc
+@@ -375,8 +375,9 @@ class WindowsEnv : public Env {
+     *result = nullptr;
+     DWORD desired_access = GENERIC_READ;
+     DWORD share_mode = FILE_SHARE_READ;
+-    ScopedHandle handle = ::CreateFileA(
+-        filename.c_str(), desired_access, share_mode,
++    auto wFilename = toUtf16(filename);
++    ScopedHandle handle = ::CreateFileW(
++        wFilename.c_str(), desired_access, share_mode,
+         /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
+         /*hTemplateFile=*/nullptr);
+     if (!handle.is_valid()) {
+@@ -392,8 +393,9 @@ class WindowsEnv : public Env {
+     *result = nullptr;
+     DWORD desired_access = GENERIC_READ;
+     DWORD share_mode = FILE_SHARE_READ;
++    auto wFilename = toUtf16(filename);
+     ScopedHandle handle =
+-        ::CreateFileA(filename.c_str(), desired_access, share_mode,
++        ::CreateFileW(wFilename.c_str(), desired_access, share_mode,
+                       /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING,
+                       FILE_ATTRIBUTE_READONLY,
+                       /*hTemplateFile=*/nullptr);
+@@ -413,11 +415,12 @@ class WindowsEnv : public Env {
+     }
+ 
+     ScopedHandle mapping =
+-        ::CreateFileMappingA(handle.get(),
+-                             /*security attributes=*/nullptr, PAGE_READONLY,
+-                             /*dwMaximumSizeHigh=*/0,
+-                             /*dwMaximumSizeLow=*/0,
+-                             /*lpName=*/nullptr);
++            ::CreateFileMappingW(handle.get(),
++                                 /*security attributes=*/nullptr,
++                                 PAGE_READONLY,
++                                 /*dwMaximumSizeHigh=*/0,
++                                 /*dwMaximumSizeLow=*/0,
++                                 /*lpName=*/nullptr);
+     if (mapping.is_valid()) {
+       void* mmap_base = ::MapViewOfFile(mapping.get(), FILE_MAP_READ,
+                                         /*dwFileOffsetHigh=*/0,
+@@ -438,8 +441,9 @@ class WindowsEnv : public Env {
+                          WritableFile** result) override {
+     DWORD desired_access = GENERIC_WRITE;
+     DWORD share_mode = 0;  // Exclusive access.
+-    ScopedHandle handle = ::CreateFileA(
+-        filename.c_str(), desired_access, share_mode,
++    auto wFilename = toUtf16(filename);
++    ScopedHandle handle = ::CreateFileW(
++        wFilename.c_str(), desired_access, share_mode,
+         /*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,
+         /*hTemplateFile=*/nullptr);
+     if (!handle.is_valid()) {
+@@ -455,8 +459,9 @@ class WindowsEnv : public Env {
+                            WritableFile** result) override {
+     DWORD desired_access = FILE_APPEND_DATA;
+     DWORD share_mode = 0;  // Exclusive access.
+-    ScopedHandle handle = ::CreateFileA(
+-        filename.c_str(), desired_access, share_mode,
++    auto wFilename = toUtf16(filename);
++    ScopedHandle handle = ::CreateFileW(
++        wFilename.c_str(), desired_access, share_mode,
+         /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
+         /*hTemplateFile=*/nullptr);
+     if (!handle.is_valid()) {
+@@ -469,14 +474,16 @@ class WindowsEnv : public Env {
+   }
+ 
+   bool FileExists(const std::string& filename) override {
+-    return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES;
++    auto wFilename = toUtf16(filename);
++    return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES;
+   }
+ 
+   Status GetChildren(const std::string& directory_path,
+                      std::vector<std::string>* result) override {
+     const std::string find_pattern = directory_path + "\\*";
+-    WIN32_FIND_DATAA find_data;
+-    HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data);
++    WIN32_FIND_DATAW find_data;
++    auto wFind_pattern = toUtf16(find_pattern);
++    HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data);
+     if (dir_handle == INVALID_HANDLE_VALUE) {
+       DWORD last_error = ::GetLastError();
+       if (last_error == ERROR_FILE_NOT_FOUND) {
+@@ -488,11 +495,12 @@ class WindowsEnv : public Env {
+       char base_name[_MAX_FNAME];
+       char ext[_MAX_EXT];
+ 
+-      if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name,
++      auto find_data_filename = toUtf8(find_data.cFileName);
++      if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, base_name,
+                         ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) {
+         result->emplace_back(std::string(base_name) + ext);
+       }
+-    } while (::FindNextFileA(dir_handle, &find_data));
++    } while (::FindNextFileW(dir_handle, &find_data));
+     DWORD last_error = ::GetLastError();
+     ::FindClose(dir_handle);
+     if (last_error != ERROR_NO_MORE_FILES) {
+@@ -501,22 +509,25 @@ class WindowsEnv : public Env {
+     return Status::OK();
+   }
+ 
+-  Status RemoveFile(const std::string& filename) override {
+-    if (!::DeleteFileA(filename.c_str())) {
++  Status DeleteFile(const std::string& filename) override {
++    auto wFilename = toUtf16(filename);
++    if (!::DeleteFileW(wFilename.c_str())) {
+       return WindowsError(filename, ::GetLastError());
+     }
+     return Status::OK();
+   }
+ 
+   Status CreateDir(const std::string& dirname) override {
+-    if (!::CreateDirectoryA(dirname.c_str(), nullptr)) {
++    auto wDirname = toUtf16(dirname);
++    if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) {
+       return WindowsError(dirname, ::GetLastError());
+     }
+     return Status::OK();
+   }
+ 
+   Status RemoveDir(const std::string& dirname) override {
+-    if (!::RemoveDirectoryA(dirname.c_str())) {
++    auto wDirname = toUtf16(dirname);
++    if (!::RemoveDirectoryW(wDirname.c_str())) {
+       return WindowsError(dirname, ::GetLastError());
+     }
+     return Status::OK();
+@@ -524,7 +535,8 @@ class WindowsEnv : public Env {
+ 
+   Status GetFileSize(const std::string& filename, uint64_t* size) override {
+     WIN32_FILE_ATTRIBUTE_DATA file_attributes;
+-    if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard,
++    auto wFilename = toUtf16(filename);
++    if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard,
+                                 &file_attributes)) {
+       return WindowsError(filename, ::GetLastError());
+     }
+@@ -538,7 +550,9 @@ class WindowsEnv : public Env {
+   Status RenameFile(const std::string& from, const std::string& to) override {
+     // Try a simple move first. It will only succeed when |to| doesn't already
+     // exist.
+-    if (::MoveFileA(from.c_str(), to.c_str())) {
++    auto wFrom = toUtf16(from);
++    auto wTo = toUtf16(to);
++    if (::MoveFileW(wFrom.c_str(), wTo.c_str())) {
+       return Status::OK();
+     }
+     DWORD move_error = ::GetLastError();
+@@ -547,7 +561,7 @@ class WindowsEnv : public Env {
+     // succeed when |to| does exist. When writing to a network share, we may not
+     // be able to change the ACLs. Ignore ACL errors then
+     // (REPLACEFILE_IGNORE_MERGE_ERRORS).
+-    if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr,
++    if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr,
+                        REPLACEFILE_IGNORE_MERGE_ERRORS,
+                        /*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) {
+       return Status::OK();
+@@ -567,8 +581,9 @@ class WindowsEnv : public Env {
+   Status LockFile(const std::string& filename, FileLock** lock) override {
+     *lock = nullptr;
+     Status result;
+-    ScopedHandle handle = ::CreateFileA(
+-        filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
++    auto wFilename = toUtf16(filename);
++    ScopedHandle handle = ::CreateFileW(
++        wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
+         /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
+         nullptr);
+     if (!handle.is_valid()) {
+@@ -608,10 +623,11 @@ class WindowsEnv : public Env {
+       return Status::OK();
+     }
+ 
+-    char tmp_path[MAX_PATH];
+-    if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) {
++    wchar_t wtmp_path[MAX_PATH];
++    if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) {
+       return WindowsError("GetTempPath", ::GetLastError());
+     }
++    std::string tmp_path = toUtf8(std::wstring(wtmp_path));
+     std::stringstream ss;
+     ss << tmp_path << "leveldbtest-" << std::this_thread::get_id();
+     *result = ss.str();
+@@ -622,7 +638,8 @@ class WindowsEnv : public Env {
+   }
+ 
+   Status NewLogger(const std::string& filename, Logger** result) override {
+-    std::FILE* fp = std::fopen(filename.c_str(), "w");
++    auto wFilename = toUtf16(filename);
++    std::FILE* fp = _wfopen(wFilename.c_str(), L"w");
+     if (fp == nullptr) {
+       *result = nullptr;
+       return WindowsError(filename, ::GetLastError());
+@@ -678,6 +695,31 @@ class WindowsEnv : public Env {
+       GUARDED_BY(background_work_mutex_);
+ 
+   Limiter mmap_limiter_;  // Thread-safe.
++
++  // Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string.
++  // See http://utf8everywhere.org/#windows
++  std::string toUtf8(const std::wstring& wstr) {
++    if (wstr.empty()) return std::string();
++    int size_needed = WideCharToMultiByte(
++      CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
++    std::string strTo(size_needed, 0);
++    WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0],
++      size_needed, NULL, NULL);
++    return strTo;
++  }
++
++  // Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character
++  // string.
++  // See http://utf8everywhere.org/#windows
++  std::wstring toUtf16(const std::string& str) {
++    if (str.empty()) return std::wstring();
++    int size_needed = MultiByteToWideChar(
++      CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0);
++    std::wstring strTo(size_needed, 0);
++    MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0],
++      size_needed);
++    return strTo;
++  }
+ };
+ 
+ // Return the maximum number of concurrent mmaps.
+diff --git a/util/env_windows_test.cc b/util/env_windows_test.cc
+index d6822d2..d108ad9 100644
+--- a/util/env_windows_test.cc
++++ b/util/env_windows_test.cc
+@@ -55,6 +55,70 @@ TEST_F(EnvWindowsTest, TestOpenOnRead) {
+   ASSERT_LEVELDB_OK(env_->RemoveFile(test_file));
+ }
+ 
++TEST_F(EnvWindowsTest, TestOpenOnRead_Unicode) {
++  // Write some test data to a single file that will be opened |n| times.
++  std::string test_dir;
++  ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
++  std::string test_file = test_dir + u8"/open_on_runðŸƒ_read.txt";
++
++  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
++  std::wstring wideUtf8Path = converter.from_bytes(test_file);
++  FILE* f = _wfopen(wideUtf8Path.c_str(), L"w");
++  ASSERT_TRUE(f != nullptr);
++  const char kFileData[] = "abcdefghijklmnopqrstuvwxyz";
++  fputs(kFileData, f);
++  fclose(f);
++
++  // Open test file some number above the sum of the two limits to force
++  // leveldb::WindowsEnv to switch from mapping the file into memory
++  // to basic file reading.
++  const int kNumFiles = kMMapLimit + 5;
++  leveldb::RandomAccessFile* files[kNumFiles] = {0};
++  for (int i = 0; i < kNumFiles; i++) {
++    ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i]));
++  }
++  char scratch;
++  Slice read_result;
++  for (int i = 0; i < kNumFiles; i++) {
++    ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch));
++    ASSERT_EQ(kFileData[i], read_result[0]);
++  }
++  for (int i = 0; i < kNumFiles; i++) {
++    delete files[i];
++  }
++  ASSERT_LEVELDB_OK(env_->DeleteFile(test_file));
++}
++
++TEST_F(EnvWindowsTest, TestGetChildrenEmpty) {
++  // Create some dummy files.
++  std::string test_dir;
++  ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
++
++  std::vector<std::string> result;
++  ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result));
++  ASSERT_EQ(2, result.size()); // "." and ".." are always returned.
++}
++
++TEST_F(EnvWindowsTest, TestGetChildren_ChildFiles) {
++  // Create some dummy files.
++  std::string test_dir;
++  ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
++
++  int childFilesCount = 10;
++  for (int i = 0; i < childFilesCount; i++) {
++    std::string test_file = test_dir + u8"/runðŸƒ_and_jumpðŸ¦˜_" + std::to_string(i) + ".txt";
++    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
++    std::wstring wTest_file = converter.from_bytes(test_file);
++    FILE* f = _wfopen(wTest_file.c_str(), L"w");
++    ASSERT_TRUE(f != nullptr);
++    fclose(f);
++  }
++
++  std::vector<std::string> result;
++  ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result));
++  ASSERT_EQ(childFilesCount + 2, result.size()); // "." and ".." are returned.
++}
++
+ }  // namespace leveldb
+ 
+ int main(int argc, char** argv) {
