honor -encoding compiler flag and defaults

Author: Martijn Hoekstra

.gitattributes

@@ -1,15 +1,16 @@
 # These files are text and should be normalized (convert crlf => lf)
-*.c       text
-*.check   text
-*.css     text
-*.html    text
-*.java    text
-*.js      text
-*.sbt     text
-*.scala   text
-*.sh      text
-*.txt     text
-*.xml     text
+
+*.c       text eol=lf
+*.check   text eol=lf
+*.css     text eol=lf
+*.html    text eol=lf
+*.java    text eol=lf
+*.js      text eol=lf
+*.sbt     text eol=lf
+*.scala   text eol=lf
+*.sh      text eol=lf
+*.txt     text eol=lf
+*.xml     text eol=lf
 
 # Windows-specific files get windows endings
 *.bat     eol=crlf

dottydoc/test/BaseTest.scala

@@ -12,7 +12,7 @@ import model.Package
 trait DottyTest {
   dotty.tools.dotc.parsing.Scanners // initialize keywords
 
-  implicit var ctx: FreshContext = {
+  implicit val ctx: FreshContext = {
     val base = new ContextBase
     import base.settings._
     val ctx = base.initialCtx.fresh

project/Build.scala

@@ -40,7 +40,7 @@ object DottyBuild extends Build {
       homepage in Global := Some(url("https://github.com/lampepfl/dotty")),
 
       // scalac options
-      scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:existentials,higherKinds,implicitConversions"),
+      scalacOptions in Global ++= Seq("-feature", "-deprecation", "-encoding", "UTF8", "-language:existentials,higherKinds,implicitConversions"),
 
       javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation")
     )

src/dotty/tools/dotc/Run.scala

@@ -9,6 +9,7 @@ import Phases._
 import Decorators._
 import dotty.tools.dotc.transform.TreeTransforms.TreeTransformer
 import io.PlainFile
+import scala.io.Codec
 import util._
 import reporting.Reporter
 import transform.TreeChecker
@@ -28,8 +29,9 @@ class Run(comp: Compiler)(implicit ctx: Context) {
   var units: List[CompilationUnit] = _
 
   def getSource(fileName: String): SourceFile = {
+    val encoding = ctx.settings.encoding.value
     val f = new PlainFile(fileName)
-    if (f.exists) new SourceFile(f)
+    if (f.exists) new SourceFile(f, Codec(encoding))
     else {
       ctx.error(s"not found: $fileName")
       NoSource
@@ -113,7 +115,7 @@ class Run(comp: Compiler)(implicit ctx: Context) {
     val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) // buffering is still advised by javadoc
     writer.write(sourceCode)
     writer.close()
-    compileSources(List(new SourceFile(virtualFile)))
+    compileSources(List(new SourceFile(virtualFile, Codec.UTF8)))
   }
 
   /** The context created for this run */

src/dotty/tools/dotc/config/Settings.scala

@@ -25,6 +25,8 @@ object Settings {
     private var values = ArrayBuffer(initialValues: _*)
     private var _wasRead: Boolean = false
 
+    override def toString = s"SettingsState(values: ${values.toList})"
+
     def value(idx: Int): Any = {
       _wasRead = true
       values(idx)

src/dotty/tools/dotc/util/SourceFile.scala

@@ -10,6 +10,7 @@ import java.io.IOException
 import Chars._
 import ScriptSourceFile._
 import Positions._
+import scala.io.Codec
 
 import java.util.Optional
 
@@ -36,9 +37,9 @@ object ScriptSourceFile {
 
 case class SourceFile(file: AbstractFile, content: Array[Char]) extends interfaces.SourceFile {
 
-  def this(_file: AbstractFile)                 = this(_file, _file.toCharArray)
-  def this(sourceName: String, cs: Seq[Char])   = this(new VirtualFile(sourceName), cs.toArray)
-  def this(file: AbstractFile, cs: Seq[Char])   = this(file, cs.toArray)
+  def this(_file: AbstractFile, codec: Codec) = this(_file, new String(_file.toByteArray, codec.charSet).toCharArray)
+  def this(sourceName: String, cs: Seq[Char]) = this(new VirtualFile(sourceName), cs.toArray)
+  def this(file: AbstractFile, cs: Seq[Char]) = this(file, cs.toArray)
 
   /** Tab increment; can be overridden */
   def tabInc = 8

test/dotc/tests.scala

@@ -23,8 +23,7 @@ class tests extends CompilerTest {
   val defaultOutputDir = "./out/"
 
   implicit val defaultOptions = noCheckOptions ++ List(
-      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases",
-      "-d", defaultOutputDir) ++ {
+      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases", "-d", defaultOutputDir) ++ {
     if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725
     else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef")
   }

test/test/CompilerTest.scala

@@ -449,21 +449,39 @@ abstract class CompilerTest {
   /** Recursively copy over source files and directories, excluding extensions
     * that aren't in extensionsToCopy. */
   private def recCopyFiles(sourceFile: Path, dest: Path): Unit = {
-    processFileDir(sourceFile, { sf =>
-      if (extensionsToCopy.contains(sf.extension)) {
-        dest.parent.jfile.mkdirs
+    
+    def copyfile(file: SFile, bytewise: Boolean): Unit = {
+      if (bytewise) {
+        val in = file.inputStream()
+        val out = SFile(dest).outputStream()
+        val buffer = new Array[Byte](1024)
+        def loop(available: Int):Unit = {
+          if (available < 0) {()}
+          else {
+            out.write(buffer, 0, available)
+            val read = in.read(buffer)
+            loop(read)
+          }
+        }
+        loop(0)
+        in.close()
+        out.close()
+      } else {
         try {
-        scala.reflect.io.File(dest)(scala.io.Codec.UTF8).writeAll(s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $sf !!!!! */", sf.slurp("UTF-8"))
+          SFile(dest)(scala.io.Codec.UTF8).writeAll(s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $file !!!!! */", file.slurp("UTF-8"))
         } catch {
-          case unmappable: java.nio.charset.UnmappableCharacterException => {
-            log(s"ERROR: reading file $sf failed to decode: $unmappable")
-            throw unmappable
-          }
-          case t: Throwable => {
-            log(s"ERROR: file $sf failed for some other reason: $t")
-            throw t
-          }
+          case unmappable: java.nio.charset.UnmappableCharacterException => 
+            copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header.
+            case unmappable: java.nio.charset.MalformedInputException => 
+            copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header.
         }
+      }
+    }
+
+    processFileDir(sourceFile, { sf =>
+      if (extensionsToCopy.contains(sf.extension)) {
+        dest.parent.jfile.mkdirs
+        copyfile(sf, false)
       } else {
         log(s"WARNING: ignoring $sf")
       }

test/test/DottyTest.scala

@@ -23,6 +23,7 @@ class DottyTest /*extends ContextEscapeDetection*/ {
     import base.settings._
     val ctx = base.initialCtx.fresh
     base.initialize()(ctx)
+    ctx.setSetting(ctx.settings.encoding, "UTF8")
     ctx
   }
 /*

test/test/ParserTest.scala

@@ -4,6 +4,7 @@ import scala.reflect.io._
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.core._
 import dotty.tools.dotc.parsing._
+import scala.io.Codec
 import Tokens._, Parsers._
 import dotty.tools.dotc.ast.untpd._
 import org.junit.Test
@@ -23,7 +24,7 @@ class ParserTest extends DottyTest {
 
   def parse(file: PlainFile): Tree = {
     //println("***** parsing " + file)
-    val source = new SourceFile(file)
+    val source = new SourceFile(file, Codec.UTF8)
     val parser = new Parser(source)
     val tree = parser.parse()
     parsed += 1

test/test/ScannerTest.scala

@@ -1,6 +1,7 @@
 package test
 
 import scala.reflect.io._
+import scala.io.Codec
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.parsing._
 import Tokens._, Scanners._
@@ -16,8 +17,8 @@ class ScannerTest extends DottyTest {
   def scan(name: String): Unit = scan(new PlainFile(name))
 
   def scan(file: PlainFile): Unit = {
-    println("***** scanning " + file)
-    val source = new SourceFile(file)
+    //println("***** scanning " + file)
+    val source = new SourceFile(file, Codec.UTF8)
     val scanner = new Scanner(source)
     var i = 0
     while (scanner.token != EOF) {

tests/run/utf16encoded.check

@@ -0,0 +1 @@
+true

tests/run/utf16encoded.flags

@@ -0,0 +1 @@
+-encoding UTF16

tests/run/utf16encoded.scala

@@ -0,0 +1 @@
+true

tests/run/utf8encoded.check

@@ -0,0 +1 @@
+-encoding UTF8

tests/run/utf8encoded.flags

@@ -0,0 +1,8 @@
+//this file is saved as UTF-8
+object UTF8Encoded {
+  def main(args: Array[String]): Unit = {
+    val testchar = '⇒'
+    println(testchar == '\u21D2')
+  }
+  
+}