[WIP] fix encoding issues

.gitattributes

@@ -1,15 +1,16 @@
 # These files are text and should be normalized (convert crlf => lf)
-*.c       text
-*.check   text
-*.css     text
-*.html    text
-*.java    text
-*.js      text
-*.sbt     text
-*.scala   text
-*.sh      text
-*.txt     text
-*.xml     text
+
+*.c       text eol=lf
+*.check   text eol=lf
+*.css     text eol=lf
+*.html    text eol=lf
+*.java    text eol=lf
+*.js      text eol=lf
+*.sbt     text eol=lf
+*.scala   text eol=lf
+*.sh      text eol=lf
+*.txt     text eol=lf
+*.xml     text eol=lf
 
 # Windows-specific files get windows endings
 *.bat     eol=crlf

dottydoc/test/BaseTest.scala

@@ -12,7 +12,7 @@ import model.Package
 trait DottyTest {
   dotty.tools.dotc.parsing.Scanners // initialize keywords
 
-  implicit var ctx: FreshContext = {
+  implicit val ctx: FreshContext = {
     val base = new ContextBase
     import base.settings._
     val ctx = base.initialCtx.fresh

project/Build.scala

@@ -40,7 +40,7 @@ object DottyBuild extends Build {
       homepage in Global := Some(url("https://github.com/lampepfl/dotty")),
 
       // scalac options
-      scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:existentials,higherKinds,implicitConversions"),
+      scalacOptions in Global ++= Seq("-feature", "-deprecation", "-encoding", "UTF8", "-language:existentials,higherKinds,implicitConversions"),
 
       javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation")
     )

src/dotty/tools/dotc/Run.scala

@@ -9,6 +9,7 @@ import Phases._
 import Decorators._
 import dotty.tools.dotc.transform.TreeTransforms.TreeTransformer
 import io.PlainFile
+import scala.io.Codec
 import util._
 import reporting.Reporter
 import transform.TreeChecker
@@ -28,8 +29,9 @@ class Run(comp: Compiler)(implicit ctx: Context) {
   var units: List[CompilationUnit] = _
 
   def getSource(fileName: String): SourceFile = {
+    val encoding = ctx.settings.encoding.value
     val f = new PlainFile(fileName)
-    if (f.exists) new SourceFile(f)
+    if (f.exists) new SourceFile(f, Codec(encoding))
     else {
       ctx.error(s"not found: $fileName")
       NoSource
@@ -113,7 +115,7 @@ class Run(comp: Compiler)(implicit ctx: Context) {
     val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) // buffering is still advised by javadoc
     writer.write(sourceCode)
     writer.close()
-    compileSources(List(new SourceFile(virtualFile)))
+    compileSources(List(new SourceFile(virtualFile, Codec.UTF8)))
   }
 
   /** The context created for this run */

src/dotty/tools/dotc/config/Settings.scala

@@ -25,6 +25,8 @@ object Settings {
     private var values = ArrayBuffer(initialValues: _*)
     private var _wasRead: Boolean = false
 
+    override def toString = s"SettingsState(values: ${values.toList})"
+
     def value(idx: Int): Any = {
       _wasRead = true
       values(idx)

src/dotty/tools/dotc/util/SourceFile.scala

@@ -10,6 +10,7 @@ import java.io.IOException
 import Chars._
 import ScriptSourceFile._
 import Positions._
+import scala.io.Codec
 
 import java.util.Optional
 
@@ -36,9 +37,9 @@ object ScriptSourceFile {
 
 case class SourceFile(file: AbstractFile, content: Array[Char]) extends interfaces.SourceFile {
 
-  def this(_file: AbstractFile)                 = this(_file, _file.toCharArray)
-  def this(sourceName: String, cs: Seq[Char])   = this(new VirtualFile(sourceName), cs.toArray)
-  def this(file: AbstractFile, cs: Seq[Char])   = this(file, cs.toArray)
+  def this(_file: AbstractFile, codec: Codec) = this(_file, new String(_file.toByteArray, codec.charSet).toCharArray)
+  def this(sourceName: String, cs: Seq[Char]) = this(new VirtualFile(sourceName), cs.toArray)
+  def this(file: AbstractFile, cs: Seq[Char]) = this(file, cs.toArray)
 
   /** Tab increment; can be overridden */
   def tabInc = 8

test/dotc/tests.scala

@@ -23,8 +23,7 @@ class tests extends CompilerTest {
   val defaultOutputDir = "./out/"
 
   implicit val defaultOptions = noCheckOptions ++ List(
-      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases",
-      "-d", defaultOutputDir) ++ {
+      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases", "-d", defaultOutputDir) ++ {
     if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725
     else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef")
   }
@@ -38,6 +37,9 @@ class tests extends CompilerTest {
   val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings")
   val scala2mode = List("-language:Scala2")
 
+  val explicitUTF8 = List("-encoding", "UTF8")
+  val explicitUTF16 = List("-encoding", "UTF16")
+
   val testsDir      = "./tests/"
   val posDir        = testsDir + "pos/"
   val posSpecialDir = testsDir + "pos-special/"
@@ -95,7 +97,7 @@ class tests extends CompilerTest {
   @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice)
   @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice)
   @Test def pos_tailcall = compileDir(posDir, "tailcall", twice)
-  @Test def pos_valueclasses = compileFiles(posDir + "valueclasses/", twice)
+  @Test def pos_valueclasses = compileFiles(posDir + "pos_valueclasses/", twice)
   @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil)
   @Test def pos_subtyping = compileFile(posDir, "subtyping", twice)
   @Test def pos_packageObj = compileFile(posDir, "i0239", twice)
@@ -118,6 +120,8 @@ class tests extends CompilerTest {
     compileFile(posSpecialDir, "spec-t5545/S_1")
     compileFile(posSpecialDir, "spec-t5545/S_2")
   }
+  @Test def pos_utf8 = compileFile(posSpecialDir, "utf8encoded", explicitUTF8)
+  @Test def pos_utf16 = compileFile(posSpecialDir, "utf16encoded", explicitUTF16)
 
   @Test def new_all = compileFiles(newDir, twice)
   @Test def repl_all = replFiles(replDir)

test/test/CompilerTest.scala

@@ -228,7 +228,7 @@ abstract class CompilerTest {
 
   private def expectedErrors(filePath: String): List[ErrorsInFile] = expectedErrors(List(filePath))
 
-  private def isNegTest(testPath: String) = testPath.contains(JFile.separator + "neg" + JFile.separator)
+  private def isNegTest(testPath: String) = testPath.contains("/neg/")
 
   private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile])
       (implicit defaultOptions: List[String]): Unit = {
@@ -413,7 +413,8 @@ abstract class CompilerTest {
     val flags = oldFlags.map(f => if (f == oldOutput) partestOutput else f) ++
       List(s"-classpath $partestOutput") // Required for separate compilation tests
 
-    getExisting(dest).isDifferent(source, flags, nerr) match {
+    val difference = getExisting(dest).isDifferent(source, flags, nerr)
+    difference match {
       case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind)
       case ExistsSame => // nothing else to do
       case ExistsDifferent =>
@@ -449,11 +450,37 @@ abstract class CompilerTest {
   /** Recursively copy over source files and directories, excluding extensions
     * that aren't in extensionsToCopy. */
   private def recCopyFiles(sourceFile: Path, dest: Path): Unit = {
-    processFileDir(sourceFile, { sf =>
+
+    def copyfile(file: SFile, bytewise: Boolean): Unit = {
+      if (bytewise) {
+        val in = file.inputStream()
+        val out = SFile(dest).outputStream()
+        val buffer = new Array[Byte](1024)
+        def loop(available: Int):Unit = {
+          if (available < 0) {()}
+          else {
+            out.write(buffer, 0, available)
+            val read = in.read(buffer)
+            loop(read)
+          }
+        }
+        loop(0)
+        in.close()
+        out.close()
+      } else {
+        try {
+          SFile(dest)(scala.io.Codec.UTF8).writeAll((s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $file !!!!! */").replace("\\", "/"), file.slurp("UTF-8"))
+        } catch {
+          case unmappable: java.nio.charset.MalformedInputException => 
+            copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header.
+        }
+      }
+    }
+
+    processFileDir(sourceFile, { sf => 
       if (extensionsToCopy.contains(sf.extension)) {
         dest.parent.jfile.mkdirs
-        dest.toFile.writeAll("/* !!!!! WARNING: DO NOT MODIFY. Original is at: $sf !!!!! */",
-          sf.slurp())
+	      copyfile(sf, false)
       } else {
         log(s"WARNING: ignoring $sf")
       }
@@ -465,7 +492,7 @@ abstract class CompilerTest {
 
   /** Reads the existing files for the given test source if any. */
   private def getExisting(dest: Path): ExistingFiles = {
-    val content: Option[Option[String]] = processFileDir(dest, f => f.safeSlurp, d => Some(""))
+    val content: Option[Option[String]] = processFileDir(dest, f => try Some(f.slurp("UTF8")) catch {case io: java.io.IOException => Some(io.toString())}, d => Some(""))
     if (content.isDefined && content.get.isDefined) {
       val flags = (dest changeExtension "flags").toFile.safeSlurp
       val nerr = (dest changeExtension "nerr").toFile.safeSlurp
@@ -479,7 +506,7 @@ abstract class CompilerTest {
       if (!genSrc.isDefined) {
         NotExists
       } else {
-        val source = processFileDir(sourceFile, { f => f.safeSlurp }, { d => Some("") },
+        val source = processFileDir(sourceFile, { f => try Some(f.slurp("UTF8")) catch {case _: java.io.IOException => None} }, { d => Some("") },
             Some("DPCompilerTest sourceFile doesn't exist: " + sourceFile)).get
         if (source == genSrc) {
           nerr match {

test/test/DottyTest.scala

@@ -23,6 +23,7 @@ class DottyTest /*extends ContextEscapeDetection*/ {
     import base.settings._
     val ctx = base.initialCtx.fresh
     base.initialize()(ctx)
+    ctx.setSetting(ctx.settings.encoding, "UTF8")
     ctx
   }
 /*

test/test/InterfaceEntryPointTest.scala

@@ -18,7 +18,7 @@ import scala.collection.mutable.ListBuffer
  */
 class InterfaceEntryPointTest {
   @Test def runCompilerFromInterface = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val mainClass = Class.forName("dotty.tools.dotc.Main")

test/test/OtherEntryPointsTest.scala

@@ -17,7 +17,7 @@ import scala.collection.mutable.ListBuffer
  */
 class OtherEntryPointsTest {
   @Test def runCompiler = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val reporter = new CustomReporter
@@ -31,7 +31,7 @@ class OtherEntryPointsTest {
   }
 
   @Test def runCompilerWithContext = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val reporter = new CustomReporter

test/test/ParserTest.scala

@@ -4,6 +4,7 @@ import scala.reflect.io._
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.core._
 import dotty.tools.dotc.parsing._
+import scala.io.Codec
 import Tokens._, Parsers._
 import dotty.tools.dotc.ast.untpd._
 import org.junit.Test
@@ -23,7 +24,7 @@ class ParserTest extends DottyTest {
 
   def parse(file: PlainFile): Tree = {
     //println("***** parsing " + file)
-    val source = new SourceFile(file)
+    val source = new SourceFile(file, Codec.UTF8)
     val parser = new Parser(source)
     val tree = parser.parse()
     parsed += 1

test/test/ScannerTest.scala

@@ -1,6 +1,7 @@
 package test
 
 import scala.reflect.io._
+import scala.io.Codec
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.parsing._
 import Tokens._, Scanners._
@@ -16,8 +17,8 @@ class ScannerTest extends DottyTest {
   def scan(name: String): Unit = scan(new PlainFile(name))
 
   def scan(file: PlainFile): Unit = {
-    println("***** scanning " + file)
-    val source = new SourceFile(file)
+    //println("***** scanning " + file)
+    val source = new SourceFile(file, Codec.UTF8)
     val scanner = new Scanner(source)
     var i = 0
     while (scanner.token != EOF) {

test/test/TestREPL.scala

@@ -34,7 +34,7 @@ class TestREPL(script: String) extends REPL {
           while (lines.hasNext && lines.head.startsWith(continuationPrompt)) {
             val continued = lines.next
             output.println(continued)
-            buf append "\n"
+            buf append System.lineSeparator()
             buf append continued.drop(continuationPrompt.length)
           }
           buf.toString
@@ -49,7 +49,7 @@ class TestREPL(script: String) extends REPL {
     out.close()
     val printed = out.toString
     val transcript = printed.drop(printed.indexOf(config.prompt))
-    if (transcript.toString != script) {
+    if (transcript.toString.lines.toList != script.lines.toList) {
       println("input differs from transcript:")
       println(transcript)
       assert(false)

tests/pos-special/utf8encoded.scala

@@ -0,0 +1,8 @@
+//this file is saved as UTF-8
+object Test {
+  def main(args: Array[String]): Unit = {
+    val testchar = '⇒'
+    println(testchar == '\u21D2')
+  }
+
+}