[Localization] NFC: Add localization serialization unit tests

unittests/Localization/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_swift_unittest(swiftLocalizationTests
-  DefToYAMLConverterTests.cpp)
+  DefToYAMLConverterTests.cpp
+  SerializationTests.cpp)
 
 target_link_libraries(swiftLocalizationTests
   PRIVATE

unittests/Localization/DefToYAMLConverterTests.cpp

@@ -10,17 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "LocalizationTest.h"
 #include "swift/Localization/LocalizationFormat.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/Signals.h"
 #include "llvm/Support/ToolOutputFile.h"
-#include "llvm/Support/YAMLParser.h"
-#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 #include <cstdlib>
@@ -30,22 +28,7 @@
 
 using namespace swift;
 using namespace swift::diag;
-
-enum LocalDiagID : uint32_t {
-#define DIAG(KIND, ID, Options, Text, Signature) ID,
-#include "swift/AST/DiagnosticsAll.def"
-  NumDiags
-};
-
-static constexpr const char *const diagnosticID[] = {
-#define DIAG(KIND, ID, Options, Text, Signature) #ID,
-#include "swift/AST/DiagnosticsAll.def"
-};
-
-static constexpr const char *const diagnosticMessages[] = {
-#define DIAG(KIND, ID, Options, Text, Signature) Text,
-#include "swift/AST/DiagnosticsAll.def"
-};
+using namespace swift::unittests;
 
 static std::string getMainExecutablePath() {
   llvm::StringRef libPath = llvm::sys::path::parent_path(SWIFTLIB_DIR);
@@ -62,50 +45,7 @@ static std::string getDefaultLocalizationPath() {
   return std::string(DefaultDiagnosticMessagesDir);
 }
 
-struct DefToYAMLConverterTest : public ::testing::Test {
-  std::string YAMLPath;
-
-public:
-  DefToYAMLConverterTest() {
-    llvm::SmallString<128> tempFilename;
-    std::error_code error =
-        llvm::sys::fs::createTemporaryFile("en", "yaml", tempFilename);
-    assert(!error);
-
-    YAMLPath = std::string(tempFilename);
-  }
-
-  void SetUp() override {
-    bool failed = convertDefIntoYAML(YAMLPath);
-    assert(!failed && "failed to generate a YAML file");
-  }
-
-  void TearDown() override { llvm::sys::fs::remove(YAMLPath); }
-
-  /// Random number in [0,n)
-  unsigned RandNumber(unsigned n) { return unsigned(rand()) % n; }
-
-protected:
-  static bool convertDefIntoYAML(std::string outputPath) {
-    std::error_code error;
-    llvm::raw_fd_ostream OS(outputPath, error, llvm::sys::fs::F_None);
-    if (OS.has_error() || error)
-      return true;
-
-    llvm::ArrayRef<const char *> ids(diagnosticID, LocalDiagID::NumDiags);
-    llvm::ArrayRef<const char *> messages(diagnosticMessages,
-                                          LocalDiagID::NumDiags);
-
-    DefToYAMLConverter converter(ids, messages);
-    converter.convert(OS);
-
-    OS.flush();
-
-    return OS.has_error();
-  }
-};
-
-TEST_F(DefToYAMLConverterTest, MissingLocalizationFiles) {
+TEST_F(LocalizationTest, MissingLocalizationFiles) {
   ASSERT_TRUE(llvm::sys::fs::exists(getDefaultLocalizationPath()));
   llvm::SmallString<128> EnglishLocalization(getDefaultLocalizationPath());
   llvm::sys::path::append(EnglishLocalization, "en");
@@ -115,15 +55,15 @@ TEST_F(DefToYAMLConverterTest, MissingLocalizationFiles) {
   ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization));
 }
 
-TEST_F(DefToYAMLConverterTest, MatchDiagnosticMessagesSequentially) {
+TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesSequentially) {
   YAMLLocalizationProducer yaml(YAMLPath);
   yaml.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) {
     llvm::StringRef msg = diagnosticMessages[static_cast<uint32_t>(id)];
     ASSERT_EQ(msg, translation);
   });
 }
 
-TEST_F(DefToYAMLConverterTest, MatchDiagnosticMessagesRandomly) {
+TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) {
   YAMLLocalizationProducer yaml(YAMLPath);
 
   std::random_device rd;

unittests/Localization/LocalizationTest.h

@@ -0,0 +1,98 @@
+//===--- LocalizationTest.h - Helper for setting up locale tests -*- C++-*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LOCALIZATION_TEST_H
+#define LOCALIZATION_TEST_H
+
+#include "swift/Localization/LocalizationFormat.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+#include <random>
+#include <string>
+#include <system_error>
+
+using namespace swift::diag;
+
+namespace swift {
+namespace unittests {
+
+enum LocalDiagID : uint32_t {
+#define DIAG(KIND, ID, Options, Text, Signature) ID,
+#include "swift/AST/DiagnosticsAll.def"
+  NumDiags
+};
+
+static constexpr const char *const diagnosticID[] = {
+#define DIAG(KIND, ID, Options, Text, Signature) #ID,
+#include "swift/AST/DiagnosticsAll.def"
+};
+
+static constexpr const char *const diagnosticMessages[] = {
+#define DIAG(KIND, ID, Options, Text, Signature) Text,
+#include "swift/AST/DiagnosticsAll.def"
+};
+
+struct LocalizationTest : public ::testing::Test {
+  std::string YAMLPath;
+
+  LocalizationTest() {
+    YAMLPath = std::string(createTemporaryFile("en", "yaml"));
+  }
+
+  void SetUp() override {
+    bool failed = convertDefIntoYAML(YAMLPath);
+    assert(!failed && "failed to generate a YAML file");
+  }
+
+  static std::string createTemporaryFile(std::string prefix,
+                                         std::string suffix) {
+    llvm::SmallString<128> tempFile;
+    std::error_code error =
+        llvm::sys::fs::createTemporaryFile(prefix, suffix, tempFile);
+    assert(!error);
+    llvm::sys::RemoveFileOnSignal(tempFile);
+    return std::string(tempFile);
+  }
+
+  /// Random number in [0,n)
+  unsigned RandNumber(unsigned n) { return unsigned(rand()) % n; }
+
+protected:
+  static bool convertDefIntoYAML(std::string outputPath) {
+    std::error_code error;
+    llvm::raw_fd_ostream OS(outputPath, error, llvm::sys::fs::F_None);
+    if (OS.has_error() || error)
+      return true;
+
+    llvm::ArrayRef<const char *> ids(diagnosticID, LocalDiagID::NumDiags);
+    llvm::ArrayRef<const char *> messages(diagnosticMessages,
+                                          LocalDiagID::NumDiags);
+
+    DefToYAMLConverter converter(ids, messages);
+    converter.convert(OS);
+
+    OS.flush();
+
+    return OS.has_error();
+  }
+};
+
+} // end namespace unittests
+} // end namespace swift
+
+#endif

unittests/Localization/SerializationTests.cpp

@@ -0,0 +1,113 @@
+//===--- LocalizationProducerTests.cpp -------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "LocalizationTest.h"
+#include "swift/Localization/LocalizationFormat.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+#include <string>
+#include <random>
+
+using namespace swift::diag;
+using namespace swift::unittests;
+
+TEST_F(LocalizationTest, TestYAMLSerialization) {
+  YAMLLocalizationProducer yaml(YAMLPath);
+
+  auto dbFile = createTemporaryFile("en", "db");
+
+  // First, let's serialize English translations
+  {
+    SerializedLocalizationWriter writer;
+
+    yaml.forEachAvailable([&writer](swift::DiagID id, llvm::StringRef translation) {
+                            writer.insert(id, translation);
+                          });
+
+    ASSERT_FALSE(writer.emit(dbFile));
+  }
+
+  // Now, let's make sure that serialized version matches "source" YAML
+  auto dbContent = llvm::MemoryBuffer::getFile(dbFile);
+  ASSERT_TRUE(dbContent);
+
+  SerializedLocalizationProducer db(std::move(dbContent.get()));
+  yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) {
+    ASSERT_EQ(translation, db.getMessageOr(id, "<no-fallback>"));
+  });
+}
+
+TEST_F(LocalizationTest, TestSerializationOfEmptyFile) {
+  auto dbFile = createTemporaryFile("by", "db");
+  SerializedLocalizationWriter writer;
+  ASSERT_FALSE(writer.emit(dbFile));
+
+  YAMLLocalizationProducer yaml(YAMLPath);
+
+  // Reading of the empty `db` file should always return default message.
+  {
+    auto dbContent = llvm::MemoryBuffer::getFile(dbFile);
+    ASSERT_TRUE(dbContent);
+
+    SerializedLocalizationProducer db(std::move(dbContent.get()));
+    yaml.forEachAvailable([&db](swift::DiagID id, llvm::StringRef translation) {
+      ASSERT_EQ("<<<default-fallback>>>",
+                db.getMessageOr(id, "<<<default-fallback>>>"));
+    });
+  }
+}
+
+TEST_F(LocalizationTest, TestSerializationWithGaps) {
+  // Initially all of the messages are included.
+  llvm::SmallBitVector includedMessages(LocalDiagID::NumDiags, true);
+
+  // Let's punch some holes in the diagnostic content.
+  for (unsigned i = 0, n = 200; i != n; ++i) {
+    unsigned position = RandNumber(LocalDiagID::NumDiags);
+    includedMessages.flip(position);
+  }
+
+  YAMLLocalizationProducer yaml(YAMLPath);
+  auto dbFile = createTemporaryFile("en", "db");
+
+  {
+    SerializedLocalizationWriter writer;
+
+    yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) {
+      if (includedMessages.test((unsigned)id))
+        writer.insert(id, translation);
+    });
+
+    ASSERT_FALSE(writer.emit(dbFile));
+  }
+
+
+  {
+    auto dbContent = llvm::MemoryBuffer::getFile(dbFile);
+    ASSERT_TRUE(dbContent);
+
+    SerializedLocalizationProducer db(std::move(dbContent.get()));
+    yaml.forEachAvailable([&](swift::DiagID id, llvm::StringRef translation) {
+      auto position = (unsigned)id;
+
+      std::string expectedMessage = includedMessages.test(position)
+                                        ? std::string(translation)
+                                        : "<<<default-fallback>>>";
+
+      ASSERT_EQ(expectedMessage, db.getMessageOr(id, "<<<default-fallback>>>"));
+    });
+  }
+}