[Localization] Add an allocator to to retain diagnostic strings

Author: HassanElDesouky

include/swift/AST/DiagnosticEngine.h

@@ -29,6 +29,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/VersionTuple.h"
 
 namespace swift {
@@ -721,6 +722,11 @@ namespace swift {
     /// diagnostic message.
     std::unique_ptr<diag::LocalizationProducer> localization;
 
+    /// This allocator will retain localized diagnostic strings for the duration
+    /// of compiler invocation.
+    llvm::BumpPtrAllocator localizationAllocator;
+    llvm::Optional<llvm::StringSaver> localizationSaver;
+
     /// The number of open diagnostic transactions. Diagnostics are only
     /// emitted once all transactions have closed.
     unsigned TransactionCount = 0;
@@ -744,7 +750,8 @@ namespace swift {
   public:
     explicit DiagnosticEngine(SourceManager &SourceMgr)
         : SourceMgr(SourceMgr), ActiveDiagnostic(),
-          TransactionStrings(TransactionAllocator) {}
+          TransactionStrings(TransactionAllocator),
+          localizationSaver(localizationAllocator) {}
 
     /// hadAnyError - return true if any *error* diagnostics have been emitted.
     bool hadAnyError() const { return state.hadAnyError(); }
@@ -804,15 +811,16 @@ namespace swift {
       if (llvm::sys::fs::exists(filePath)) {
         if (auto file = llvm::MemoryBuffer::getFile(filePath)) {
           localization = std::make_unique<diag::SerializedLocalizationProducer>(
-              std::move(file.get()), getPrintDiagnosticNames());
+              std::move(file.get()), localizationSaver,
+              getPrintDiagnosticNames());
         }
       } else {
         llvm::sys::path::replace_extension(filePath, ".yaml");
         // In case of missing localization files, we should fallback to messages
         // from `.def` files.
         if (llvm::sys::fs::exists(filePath)) {
           localization = std::make_unique<diag::YAMLLocalizationProducer>(
-              filePath.str(), getPrintDiagnosticNames());
+              filePath.str(), localizationSaver, getPrintDiagnosticNames());
         }
       }
     }

include/swift/Localization/LocalizationFormat.h

@@ -26,6 +26,7 @@
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/OnDiskHashTable.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -154,12 +155,15 @@ class SerializedLocalizationWriter {
 };
 
 class LocalizationProducer {
+  llvm::Optional<llvm::StringSaver> localizationSaver;
   bool printDiagnosticName;
-  std::string localizedDebugDiagnosticMessage;
 
 public:
-  LocalizationProducer(bool printDiagnosticName = false)
-      : printDiagnosticName(printDiagnosticName) {}
+  LocalizationProducer(llvm::Optional<llvm::StringSaver> localizationSaver =
+                           llvm::Optional<llvm::StringSaver>(),
+                       bool printDiagnosticName = false)
+      : localizationSaver(localizationSaver),
+        printDiagnosticName(printDiagnosticName) {}
 
   /// If the  message isn't available/localized in current context
   /// return the fallback default message.
@@ -180,8 +184,11 @@ class YAMLLocalizationProducer final : public LocalizationProducer {
 public:
   /// The diagnostics IDs that are no longer available in `.def`
   std::vector<std::string> unknownIDs;
-  explicit YAMLLocalizationProducer(llvm::StringRef filePath,
-                                    bool printDiagnosticName = false);
+  explicit YAMLLocalizationProducer(
+      llvm::StringRef filePath,
+      llvm::Optional<llvm::StringSaver> localizationSaver =
+          llvm::Optional<llvm::StringSaver>(),
+      bool printDiagnosticName = false);
 
   /// Iterate over all of the available (non-empty) translations
   /// maintained by this producer, callback gets each translation
@@ -203,6 +210,8 @@ class SerializedLocalizationProducer final : public LocalizationProducer {
 public:
   explicit SerializedLocalizationProducer(
       std::unique_ptr<llvm::MemoryBuffer> buffer,
+      llvm::Optional<llvm::StringSaver> localizationSaver =
+          llvm::Optional<llvm::StringSaver>(),
       bool printDiagnosticName = false);
 
 protected:

lib/Localization/LocalizationFormat.cpp

@@ -98,15 +98,20 @@ LocalizationProducer::getMessageOr(swift::DiagID id,
   if (localizedMessage.empty())
     return defaultMessage;
   llvm::StringRef diagnosticName(diagnosticNameStrings[(unsigned)id]);
-  localizedDebugDiagnosticMessage =
-      localizedMessage.str() + diagnosticName.str();
-  return printDiagnosticName ? localizedDebugDiagnosticMessage
-                             : localizedMessage;
+  if (localizationSaver && printDiagnosticName) {
+    auto localizedDebugDiagnosticMessage =
+        localizationSaver->save(localizedMessage.str() + diagnosticName.str());
+    return localizedDebugDiagnosticMessage;
+  }
+  return localizedMessage;
 }
 
 SerializedLocalizationProducer::SerializedLocalizationProducer(
-    std::unique_ptr<llvm::MemoryBuffer> buffer, bool printDiagnosticName)
-    : LocalizationProducer(printDiagnosticName), Buffer(std::move(buffer)) {
+    std::unique_ptr<llvm::MemoryBuffer> buffer,
+    llvm::Optional<llvm::StringSaver> localizationSaver,
+    bool printDiagnosticName)
+    : LocalizationProducer(localizationSaver, printDiagnosticName),
+      Buffer(std::move(buffer)) {
   auto base =
       reinterpret_cast<const unsigned char *>(Buffer.get()->getBufferStart());
   auto tableOffset = endian::read<offset_type>(base, little);
@@ -122,9 +127,11 @@ SerializedLocalizationProducer::getMessage(swift::DiagID id) const {
   return {(const char *)value.getDataPtr(), value.getDataLen()};
 }
 
-YAMLLocalizationProducer::YAMLLocalizationProducer(llvm::StringRef filePath,
-                                                   bool printDiagnosticName)
-    : LocalizationProducer(printDiagnosticName) {
+YAMLLocalizationProducer::YAMLLocalizationProducer(
+    llvm::StringRef filePath,
+    llvm::Optional<llvm::StringSaver> localizationSaver,
+    bool printDiagnosticName)
+    : LocalizationProducer(localizationSaver, printDiagnosticName) {
   auto FileBufOrErr = llvm::MemoryBuffer::getFileOrSTDIN(filePath);
   llvm::MemoryBuffer *document = FileBufOrErr->get();
   diag::LocalizationInput yin(document->getBuffer());

test/diagnostics/Localization/en_localization.swift

@@ -1,12 +1,18 @@
-// RUN: %target-typecheck-verify-swift -localization-path %S/Inputs -locale en
+// RUN: not %target-swift-frontend -typecheck %s 2>&1 -localization-path %S/Inputs -locale en | %FileCheck %s --check-prefix=CHECK_NONAMES
+// RUN: not %target-swift-frontend -debug-diagnostic-names -localization-path %S/Inputs -locale en -typecheck %s 2>&1 | %FileCheck %s --check-prefix=CHECK_NAMES
 
 _ = "HI!
-// expected-error@-1{{unterminated string literal}}
+// CHECK_NONAMES: error: unterminated string literal{{$}}
+// CHECK_NAMES: error: unterminated string literal [lex_unterminated_string]{{$}}
+
 var self1 = self1
-// expected-note@-1 2{{through reference here}}
-// expected-error@-2 {{circular reference}}
+// CHECK_NONAMES: error: circular reference{{$}}
+// CHECK_NONAMES: note: through reference here{{$}}
+// CHECK_NAMES: error: circular reference [circular_reference]{{$}}
+// CHECK_NAMES: note: through reference here [circular_reference_through]{{$}}
 
 struct Broken {
-  var b : Bool = True // expected-error{{cannot find 'True' in scope}}
+  var b : Bool = True 
 }
+// CHECK_NONAMES: error: cannot find 'True' in scope{{$}}
 // CHECK_NAMES: error: cannot find 'True' in scope [cannot_find_in_scope]{{$}}

test/diagnostics/Localization/fr_localization.swift

@@ -1,18 +1,24 @@
 // RUN: %empty-directory(%t)
 // RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/
 // RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/ 2>&1 | %FileCheck %s
-// RUN: %target-typecheck-verify-swift -localization-path %t -locale fr
+// RUN: not %target-swift-frontend -typecheck %s 2>&1 -localization-path %S/Inputs -locale fr | %FileCheck %s --check-prefix=CHECK_NONAMES
+// RUN: not %target-swift-frontend -debug-diagnostic-names -localization-path %S/Inputs -locale fr -typecheck %s 2>&1 | %FileCheck %s --check-prefix=CHECK_NAMES
 
 // CHECK: These diagnostic IDs are no longer availiable: 'not_available_in_def, not_available_in_def_2, not_available_in_def_3, not_available_in_def_4, not_available_in_def_5'
 _ = "HI!
-// expected-error@-1{{chaîne non terminée littérale}}
+// CHECK_NONAMES: error: chaîne non terminée littérale{{$}}
+// CHECK_NAMES: error: chaîne non terminée littérale [lex_unterminated_string]{{$}}
 
 // FIXME: This used to produce a localized diagnostic.
 
-var self1 = self1 // expected-note 2{{through reference here}}
-// expected-error@-1 {{circular reference}}
+var self1 = self1
+// CHECK_NONAMES: error: circular reference{{$}}
+// CHECK_NONAMES: note: through reference here{{$}}
+// CHECK_NAMES: error: circular reference [circular_reference]{{$}}
+// CHECK_NAMES: note: through reference here [circular_reference_through]{{$}}
 
 struct Broken {
-  var b : Bool = True // expected-error{{impossible de trouver 'True' portée}}
+  var b : Bool = True
 }
+// CHECK_NONAMES: error: impossible de trouver 'True' portée{{$}}
 // CHECK_NAMES: error: impossible de trouver 'True' portée [cannot_find_in_scope]{{$}}