Merge pull request #32630 from HassanElDesouky/LocaleFlagsRE

Reopen [Diag] Create frontend flags for localization

Author: davezarzycki

include/swift/AST/DiagnosticEngine.h

@@ -740,9 +740,10 @@ namespace swift {
     }
 
     void setLocalization(std::string locale, std::string path) {
-      if (!locale.empty() && !path.empty())
-        localization =
-            std::make_unique<diag::YAMLLocalizationProducer>(locale, path);
+      assert(!locale.empty());
+      assert(!path.empty());
+      localization =
+          std::make_unique<diag::YAMLLocalizationProducer>(locale, path);
     }
 
     void ignoreDiagnostic(DiagID id) {

include/swift/AST/DiagnosticsFrontend.def

@@ -76,6 +76,13 @@ ERROR(error_unknown_arg,none,
       "unknown argument: '%0'", (StringRef))
 ERROR(error_invalid_arg_value,none,
       "invalid value '%1' in '%0'", (StringRef, StringRef))
+WARNING(warning_invalid_locale_code,none,
+      "unsupported locale code; supported locale codes are: '%0'", (StringRef))
+WARNING(warning_locale_path_not_found,none,
+      "specified localization directory '%0' does not exist, "
+      "translation is disabled", (StringRef))
+WARNING(warning_cannot_find_locale_file,none,
+      "cannot find translations for '%0' at '%1': no such file", (StringRef, StringRef))
 WARNING(warning_cannot_multithread_batch_mode,none,
       "ignoring -num-threads argument; cannot multithread batch mode", ())
 ERROR(error_unsupported_option_argument,none,

include/swift/AST/LocalizationLanguages.def

@@ -0,0 +1,207 @@
+//===--- LocalizationLanguages.def - Localization Languages ----*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file has all language codes defined by ISO 639-1 for localization.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SUPPORTED_LOCALE
+#define SUPPORTED_LOCALE(Code, Language)
+#endif
+
+SUPPORTED_LOCALE(ab, Abkhazian)
+SUPPORTED_LOCALE(aa, Afar)
+SUPPORTED_LOCALE(af, Afrikaans)
+SUPPORTED_LOCALE(ak, Akan)
+SUPPORTED_LOCALE(sq, Albanian)
+SUPPORTED_LOCALE(am, Amharic)
+SUPPORTED_LOCALE(ar, Arabic)
+SUPPORTED_LOCALE(an, Aragonese)
+SUPPORTED_LOCALE(hy, Armenian)
+SUPPORTED_LOCALE(as, Assamese)
+SUPPORTED_LOCALE(av, Avaric)
+SUPPORTED_LOCALE(ae, Avestan)
+SUPPORTED_LOCALE(ay, Aymara)
+SUPPORTED_LOCALE(az, Azerbaijani)
+SUPPORTED_LOCALE(bm, Bambara)
+SUPPORTED_LOCALE(ba, Bashkir)
+SUPPORTED_LOCALE(eu, Basque)
+SUPPORTED_LOCALE(be, Belarusian)
+SUPPORTED_LOCALE(bn, Bengali)
+SUPPORTED_LOCALE(bh, Bihari languages)
+SUPPORTED_LOCALE(bi, Bislama)
+SUPPORTED_LOCALE(bs, Bosnian)
+SUPPORTED_LOCALE(br, Breton)
+SUPPORTED_LOCALE(bg, Bulgarian)
+SUPPORTED_LOCALE(my, Burmese)
+SUPPORTED_LOCALE(ca, Catalan)
+SUPPORTED_LOCALE(km, Central Khmer)
+SUPPORTED_LOCALE(ch, Chamorro)
+SUPPORTED_LOCALE(ce, Chechen)
+SUPPORTED_LOCALE(ny, Chichewa)
+SUPPORTED_LOCALE(zh, Chinese)
+SUPPORTED_LOCALE(cu, Church Slavonic)
+SUPPORTED_LOCALE(cv, Chuvash)
+SUPPORTED_LOCALE(kw, Cornish)
+SUPPORTED_LOCALE(co, Corsican)
+SUPPORTED_LOCALE(cr, Cree)
+SUPPORTED_LOCALE(hr, Croatian)
+SUPPORTED_LOCALE(cs, Czech)
+SUPPORTED_LOCALE(da, Danish)
+SUPPORTED_LOCALE(dv, Divehi)
+SUPPORTED_LOCALE(nl, Dutch)
+SUPPORTED_LOCALE(dz, Dzongkha)
+SUPPORTED_LOCALE(en, English)
+SUPPORTED_LOCALE(eo, Esperanto)
+SUPPORTED_LOCALE(et, Estonian)
+SUPPORTED_LOCALE(ee, Ewe)
+SUPPORTED_LOCALE(fo, Faroese)
+SUPPORTED_LOCALE(fj, Fijian)
+SUPPORTED_LOCALE(fi, Finnish)
+SUPPORTED_LOCALE(fr, French)
+SUPPORTED_LOCALE(ff, Fulah)
+SUPPORTED_LOCALE(gd, Gaelic)
+SUPPORTED_LOCALE(gl, Galician)
+SUPPORTED_LOCALE(lg, Ganda)
+SUPPORTED_LOCALE(ka, Georgian)
+SUPPORTED_LOCALE(de, German)
+SUPPORTED_LOCALE(ki, Kikuyu)
+SUPPORTED_LOCALE(el, Greek)
+SUPPORTED_LOCALE(kl, Kalaallisut)
+SUPPORTED_LOCALE(gn, Guarani)
+SUPPORTED_LOCALE(gu, Gujarati)
+SUPPORTED_LOCALE(ht, Haitian Creole)
+SUPPORTED_LOCALE(ha, Hausa)
+SUPPORTED_LOCALE(he, Hebrew)
+SUPPORTED_LOCALE(hz, Herero)
+SUPPORTED_LOCALE(hi, Hindi)
+SUPPORTED_LOCALE(ho, Hiri Motu)
+SUPPORTED_LOCALE(hu, Hungarian)
+SUPPORTED_LOCALE(is, Icelandic)
+SUPPORTED_LOCALE(io, Ido)
+SUPPORTED_LOCALE(ig, Igbo)
+SUPPORTED_LOCALE(id, Indonesian)
+SUPPORTED_LOCALE(ia, Interlingua)
+SUPPORTED_LOCALE(ie, Interlingue)
+SUPPORTED_LOCALE(iu, Inuktitut)
+SUPPORTED_LOCALE(ik, Inupiaq)
+SUPPORTED_LOCALE(ga, Irish)
+SUPPORTED_LOCALE(it, Italian)
+SUPPORTED_LOCALE(ja, Japanese)
+SUPPORTED_LOCALE(jv, Javanese)
+SUPPORTED_LOCALE(kn, Kannada)
+SUPPORTED_LOCALE(kr, Kanuri)
+SUPPORTED_LOCALE(ks, Kashmiri)
+SUPPORTED_LOCALE(kk, Kazakh)
+SUPPORTED_LOCALE(rw, Kinyarwanda)
+SUPPORTED_LOCALE(kv, Komi)
+SUPPORTED_LOCALE(kg, Kongo)
+SUPPORTED_LOCALE(ko, Korean)
+SUPPORTED_LOCALE(kj, Kwanyama)
+SUPPORTED_LOCALE(ku, Kurdish)
+SUPPORTED_LOCALE(ky, Kyrgyz)
+SUPPORTED_LOCALE(lo, Lao)
+SUPPORTED_LOCALE(la, Latin)
+SUPPORTED_LOCALE(lv, Latvian)
+SUPPORTED_LOCALE(lb, Letzeburgesch)
+SUPPORTED_LOCALE(li, Limburgish)
+SUPPORTED_LOCALE(ln, Lingala)
+SUPPORTED_LOCALE(lt, Lithuanian)
+SUPPORTED_LOCALE(lu, Luba-Katanga)
+SUPPORTED_LOCALE(mk, Macedonian)
+SUPPORTED_LOCALE(mg, Malagasy)
+SUPPORTED_LOCALE(ms, Malay)
+SUPPORTED_LOCALE(ml, Malayalam)
+SUPPORTED_LOCALE(mt, Maltese)
+SUPPORTED_LOCALE(gv, Manx)
+SUPPORTED_LOCALE(mi, Maori)
+SUPPORTED_LOCALE(mr, Marathi)
+SUPPORTED_LOCALE(mh, Marshallese)
+SUPPORTED_LOCALE(ro, Romanian)
+SUPPORTED_LOCALE(mn, Mongolian)
+SUPPORTED_LOCALE(na, Nauru)
+SUPPORTED_LOCALE(nv, Navajo)
+SUPPORTED_LOCALE(nd, Northern Ndebele)
+SUPPORTED_LOCALE(ng, Ndonga)
+SUPPORTED_LOCALE(ne, Nepali)
+SUPPORTED_LOCALE(se, Northern Sami)
+SUPPORTED_LOCALE(no, Norwegian)
+SUPPORTED_LOCALE(nb, Norwegian Bokmål)
+SUPPORTED_LOCALE(nn, Norwegian Nynorsk)
+SUPPORTED_LOCALE(ii, Nuosu Sichuan Yi)
+SUPPORTED_LOCALE(oc, Occitan)
+SUPPORTED_LOCALE(oj, Ojibwa)
+SUPPORTED_LOCALE(or, Oriya)
+SUPPORTED_LOCALE(om, Oromo)
+SUPPORTED_LOCALE(os, Ossetian)
+SUPPORTED_LOCALE(pi, Pali)
+SUPPORTED_LOCALE(pa, Panjabi)
+SUPPORTED_LOCALE(ps, Pashto)
+SUPPORTED_LOCALE(fa, Persian)
+SUPPORTED_LOCALE(pl, Polish)
+SUPPORTED_LOCALE(pt, Portuguese)
+SUPPORTED_LOCALE(qu, Quechua)
+SUPPORTED_LOCALE(rm, Romansh)
+SUPPORTED_LOCALE(rn, Rundi)
+SUPPORTED_LOCALE(ru, Russian)
+SUPPORTED_LOCALE(sm, Samoan)
+SUPPORTED_LOCALE(sg, Sango)
+SUPPORTED_LOCALE(sa, Sanskrit)
+SUPPORTED_LOCALE(sc, Sardinian)
+SUPPORTED_LOCALE(sr, Serbian)
+SUPPORTED_LOCALE(sn, Shona)
+SUPPORTED_LOCALE(sd, Sindhi)
+SUPPORTED_LOCALE(si, Sinhala Sinhalese)
+SUPPORTED_LOCALE(sk, Slovak)
+SUPPORTED_LOCALE(sl, Slovenian)
+SUPPORTED_LOCALE(so, Somali)
+SUPPORTED_LOCALE(st, Sotho Southern)
+SUPPORTED_LOCALE(nr, South Ndebele)
+SUPPORTED_LOCALE(es, Spanish Castilian)
+SUPPORTED_LOCALE(su, Sundanese)
+SUPPORTED_LOCALE(sw, Swahili)
+SUPPORTED_LOCALE(ss, Swati)
+SUPPORTED_LOCALE(sv, Swedish)
+SUPPORTED_LOCALE(tl, Tagalog)
+SUPPORTED_LOCALE(ty, Tahitian)
+SUPPORTED_LOCALE(tg, Tajik)
+SUPPORTED_LOCALE(ta, Tamil)
+SUPPORTED_LOCALE(tt, Tatar)
+SUPPORTED_LOCALE(te, Telugu)
+SUPPORTED_LOCALE(th, Thai)
+SUPPORTED_LOCALE(bo, Tibetan)
+SUPPORTED_LOCALE(ti, Tigrinya)
+SUPPORTED_LOCALE(to, Tonga)
+SUPPORTED_LOCALE(ts, Tsonga)
+SUPPORTED_LOCALE(tn, Tswana)
+SUPPORTED_LOCALE(tr, Turkish)
+SUPPORTED_LOCALE(tk, Turkmen)
+SUPPORTED_LOCALE(tw, Twi)
+SUPPORTED_LOCALE(ug, Uighur)
+SUPPORTED_LOCALE(uk, Ukrainian)
+SUPPORTED_LOCALE(ur, Urdu)
+SUPPORTED_LOCALE(uz, Uzbek)
+SUPPORTED_LOCALE(ve, Venda)
+SUPPORTED_LOCALE(vi, Vietnamese)
+SUPPORTED_LOCALE(vo, Volap_k)
+SUPPORTED_LOCALE(wa, Walloon)
+SUPPORTED_LOCALE(cy, Welsh)
+SUPPORTED_LOCALE(fy, Western Frisian)
+SUPPORTED_LOCALE(wo, Wolof)
+SUPPORTED_LOCALE(xh, Xhosa)
+SUPPORTED_LOCALE(yi, Yiddish)
+SUPPORTED_LOCALE(yo, Yoruba)
+SUPPORTED_LOCALE(za, Zhuang Chuang)
+SUPPORTED_LOCALE(zu, Zulu)
+
+#undef SUPPORTED_LOCALE
+

include/swift/Basic/DiagnosticOptions.h

@@ -67,6 +67,11 @@ class DiagnosticOptions {
 
   std::string DiagnosticDocumentationPath = "";
 
+  std::string LocalizationCode = "";
+
+  // Diagnostic messages directory path.
+  std::string LocalizationPath = "";
+
   /// Return a hash code of any components from these options that should
   /// contribute to a Swift Bridging PCH hash.
   llvm::hash_code getPCHHashComponents() const {

include/swift/Option/Options.td

@@ -389,6 +389,14 @@ def diagnostic_style : Separate<["-"], "diagnostic-style">,
 def diagnostic_style_EQ : Joined<["-"], "diagnostic-style=">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   MetaVarName<"<style>">, Alias<diagnostic_style>;
+def locale : Separate<["-"], "locale">,
+  Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
+  HelpText<"Choose a language for diagnostic messages">,
+  MetaVarName<"<locale-code>">;
+def localization_path : Separate<["-"], "localization-path">,
+  Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,
+  HelpText<"Path to localized diagnostic messages directory">,
+  MetaVarName<"<path>">;
 
 def module_cache_path : Separate<["-"], "module-cache-path">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,

lib/Driver/ToolChains.cpp

@@ -242,6 +242,8 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI,
   inputArgs.AddLastArg(arguments,
                        options::OPT_emit_fine_grained_dependency_sourcefile_dot_files);
   inputArgs.AddLastArg(arguments, options::OPT_package_description_version);
+  inputArgs.AddLastArg(arguments, options::OPT_locale);
+  inputArgs.AddLastArg(arguments, options::OPT_localization_path);
   inputArgs.AddLastArg(arguments, options::OPT_serialize_diagnostics_path);
   inputArgs.AddLastArg(arguments, options::OPT_debug_diagnostic_names);
   inputArgs.AddLastArg(arguments, options::OPT_print_educational_notes);

lib/Frontend/CompilerInvocation.cpp

@@ -34,6 +34,11 @@ using namespace llvm::opt;
 /// The path for Swift libraries in the OS on Darwin.
 #define DARWIN_OS_LIBRARY_PATH "/usr/lib/swift"
 
+static constexpr const char *const localeCodes[] = {
+#define SUPPORTED_LOCALE(Code, Language) #Code,
+#include "swift/AST/LocalizationLanguages.def"
+};
+
 swift::CompilerInvocation::CompilerInvocation() {
   setTargetTriple(llvm::sys::getDefaultTargetTriple());
 }
@@ -57,6 +62,14 @@ void CompilerInvocation::setMainExecutablePath(StringRef Path) {
   llvm::sys::path::append(DiagnosticDocsPath, "share", "doc", "swift",
                           "diagnostics");
   DiagnosticOpts.DiagnosticDocumentationPath = std::string(DiagnosticDocsPath.str());
+
+  // Compute the path of the YAML diagnostic messages directory files
+  // in the toolchain.
+  llvm::SmallString<128> DiagnosticMessagesDir(Path);
+  llvm::sys::path::remove_filename(DiagnosticMessagesDir); // Remove /swift
+  llvm::sys::path::remove_filename(DiagnosticMessagesDir); // Remove /bin
+  llvm::sys::path::append(DiagnosticMessagesDir, "share", "swift");
+  DiagnosticOpts.LocalizationPath = std::string(DiagnosticMessagesDir.str());
 }
 
 void CompilerInvocation::setDefaultPrebuiltCacheIfNecessary() {
@@ -928,6 +941,43 @@ static bool ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   if (Arg *A = Args.getLastArg(OPT_diagnostic_documentation_path)) {
     Opts.DiagnosticDocumentationPath = A->getValue();
   }
+  if (Arg *A = Args.getLastArg(OPT_locale)) {
+    std::string localeCode = A->getValue();
+
+    // Check if the locale code is available.
+    if (llvm::none_of(localeCodes, [&](const char *locale) {
+          return localeCode == locale;
+        })) {
+      std::string availableLocaleCodes = "";
+      llvm::interleave(
+          std::begin(localeCodes), std::end(localeCodes),
+          [&](std::string locale) { availableLocaleCodes += locale; },
+          [&] { availableLocaleCodes += ", "; });
+
+      Diags.diagnose(SourceLoc(), diag::warning_invalid_locale_code,
+                     availableLocaleCodes);
+    } else {
+      Opts.LocalizationCode = localeCode;
+    }
+  }
+  if (Arg *A = Args.getLastArg(OPT_localization_path)) {
+    if (!llvm::sys::fs::exists(A->getValue())) {
+      Diags.diagnose(SourceLoc(), diag::warning_locale_path_not_found,
+                     A->getValue());
+    } else if (!Opts.LocalizationCode.empty()) {
+      // Check if the localization path exists but it doesn't have a file
+      // for the specified locale code.
+      llvm::SmallString<128> localizationPath(A->getValue());
+      llvm::sys::path::append(localizationPath, Opts.LocalizationCode);
+      llvm::sys::path::replace_extension(localizationPath, ".yaml");
+      if (!llvm::sys::fs::exists(localizationPath)) {
+        Diags.diagnose(SourceLoc(), diag::warning_cannot_find_locale_file,
+                       Opts.LocalizationCode, localizationPath);
+      }
+
+      Opts.LocalizationPath = A->getValue();
+    }
+  }
   assert(!(Opts.WarningsAsErrors && Opts.SuppressWarnings) &&
          "conflicting arguments; should have been caught by driver");
 

lib/Frontend/Frontend.cpp

@@ -410,6 +410,11 @@ void CompilerInstance::setUpDiagnosticOptions() {
   }
   Diagnostics.setDiagnosticDocumentationPath(
       Invocation.getDiagnosticOptions().DiagnosticDocumentationPath);
+  if (!Invocation.getDiagnosticOptions().LocalizationCode.empty()) {
+    Diagnostics.setLocalization(
+        Invocation.getDiagnosticOptions().LocalizationCode,
+        Invocation.getDiagnosticOptions().LocalizationPath);
+  }
 }
 
 // The ordering of ModuleLoaders is important!

test/diagnostics/Localization/Inputs/en.yaml

@@ -0,0 +1,30 @@
+#===--- en.yaml - Localized diagnostic messages for English ---*- YAML -*-===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===----------------------------------------------------------------------===#
+#
+# This file defines the diagnostic messages for the English language.
+# Each diagnostic is described in the following format:
+# - id: "<diagnostic-id>"
+#   msg: "<diagnostic-message>"
+# 
+#===----------------------------------------------------------------------===#
+
+- id: "lex_unterminated_string"
+  msg: "unterminated string literal"
+  
+- id: "var_init_self_referential"
+  msg: "variable used within its own initial value"
+
+- id: "cannot_find_in_scope"
+  msg: "cannot %select{find|find operator}1 %0 in scope"
+
+- id: "warning_invalid_locale_code"
+  msg: "unsupported locale code; supported locale codes are '%0'"