[ConstraintSystem] Add a new stat to be used for expression type checker performance testing.

This counts the number of leaf scopes we reach while solving the
constraint sytem, and is a much better measure of the growth of
unnecessary work than the total number of scopes opened.

There were two tests where I had a difficult time getting scale-test
to fit the curve even after adjusting some of the parameters, so I've
left those to use the old stat for now.

Author: rudkx

include/swift/Basic/Statistic.h

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -28,6 +28,13 @@
     ++FStat;                                                            \
   } while (0)
 
+#define SWIFT_FUNC_STAT_INCREMENT(x)                                    \
+  do {                                                                  \
+    static llvm::Statistic FStat =                                      \
+      {DEBUG_TYPE, __func__, __func__, {0}, {false}};                   \
+    FStat += (x);                                                       \
+  } while (0)
+
 // Helper class designed to consolidate reporting of LLVM statistics and timers
 // across swift compilations that typically invoke many drivers, each running
 // many frontends. Additionally collects some cheap "always-on" statistics,

lib/Sema/CSSolver.cpp

@@ -1240,7 +1240,6 @@ bool ConstraintSystem::solve(Expr *const expr,
     auto &log = getASTContext().TypeCheckerDebug->getStream();
     log << "---Solver statistics---\n";
     log << "Total number of scopes explored: " << solverState->NumStatesExplored << "\n";
-    log << "Number of leaf scopes explored: " << solverState->leafScopes << "\n";
     log << "Maximum depth reached while exploring solutions: " << solverState->maxDepth << "\n";
     if (Timer) {
       auto timeInMillis =

lib/Sema/CSStep.cpp

@@ -76,6 +76,12 @@ StepResult SplitterStep::resume(bool prevFailed) {
   for (auto &component : Components)
     workList.splice(workList.end(), component);
 
+  // We want to compute the number of leaf scopes as if the system had
+  // not been split, so allow each additional component after the
+  // first to have a free leaf scope that doesn't count against the
+  // total for the system.
+  CS.decrementLeafScopes(Components.size() - 1);
+
   // If we came back to this step and previous (one of the components)
   // failed, it means that we can't solve this step either.
   if (prevFailed)

lib/Sema/ConstraintSystem.cpp

@@ -90,6 +90,19 @@ void ConstraintSystem::incrementScopeCounter() {
     TC.Context.Stats->getFrontendCounters().NumConstraintScopes++;
 }
 
+void ConstraintSystem::numLeafScopes(unsigned value) {
+  // NOTE: We rely on 2's complement addition here.
+  SWIFT_FUNC_STAT_INCREMENT(value);
+}
+
+void ConstraintSystem::incrementLeafScopes(unsigned increment) {
+  numLeafScopes(increment);
+}
+
+void ConstraintSystem::decrementLeafScopes(unsigned decrement) {
+  numLeafScopes(-decrement);
+}
+
 bool ConstraintSystem::hasFreeTypeVariables() {
   // Look for any free type variables.
   for (auto tv : TypeVariables) {

lib/Sema/ConstraintSystem.h

@@ -1166,12 +1166,6 @@ class ConstraintSystem {
     /// \brief Maximum depth reached so far in exploring solutions.
     unsigned maxDepth = 0;
 
-    /// \brief Count of the number of leaf scopes we've created. These
-    /// either result in a failure to solve, or in a solution, unlike
-    /// all the intermediate scopes. They are interesting to track as
-    /// part of a metric of whether an expression is too complex.
-    unsigned leafScopes = 0;
-
     /// \brief Whether to record failures or not.
     bool recordFixes = false;
 
@@ -1268,7 +1262,7 @@ class ConstraintSystem {
 
       unsigned countScopesExplored = NumStatesExplored - scope->scopeNumber;
       if (countScopesExplored == 1)
-        ++leafScopes;
+        CS.incrementLeafScopes();
 
       SolverScope *savedScope;
       // The position of last retired constraint before given scope.
@@ -1439,6 +1433,10 @@ class ConstraintSystem {
 
   void incrementScopeCounter();
 
+  void numLeafScopes(unsigned adjustment);
+  void incrementLeafScopes(unsigned increment =1);
+  void decrementLeafScopes(unsigned decrement =1);
+
 public:
   /// \brief Introduces a new solver scope, which any changes to the
   /// solver state or constraint system are temporary and will be undone when

validation-test/Sema/type_checker_perf/fast/array_of_tuples.swift.gyb

@@ -0,0 +1,9 @@
+// RUN: %scale-test --begin 1 --end 20 --step 1 --select numLeafScopes %s
+// REQUIRES: OS=macosx
+// REQUIRES: asserts
+
+let a = [
+%for i in range(0, N):
+  (1, 1),
+%end
+]

validation-test/Sema/type_checker_perf/fast/more_specialized_generic_func.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 8 --end 40 --step 2 --select incrementScopeCounter %s --expected-exit-code 0
+// RUN: %scale-test --begin 8 --end 40 --step 2 --select numLeafScopes %s --expected-exit-code 0
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar18699199.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 10 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 public enum E

validation-test/Sema/type_checker_perf/fast/rdar18724501.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar19181998.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 30 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 30 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar19181998_nominals.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 30 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 30 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar19394804.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 10 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar19738292.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 7 --end 12 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 7 --end 12 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 // REQUIRES: rdar42650365

validation-test/Sema/type_checker_perf/fast/rdar19777895.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 3 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 3 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar19915443.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 7 --end 15 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 7 --end 15 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 let a = [0]

validation-test/Sema/type_checker_perf/fast/rdar20233198_any.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar20233198_explicit_overloads.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar20233198_named.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar20233198_typed.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar20233198_weak.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s -Xfrontend=-verify
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s -Xfrontend=-verify
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar21070413.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar21328584.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar21720888.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 10 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar21930551.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar22532650.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 10 --end 15 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 10 --end 15 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar22626740.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar22810685.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar22836718.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 5 --end 10 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 5 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar23327871.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 8 --end 16 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 16 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar25866240.swift.gyb

@@ -2,10 +2,11 @@
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 
-func f() {
-% for i in range(N):
-  let collection${i} = [String]()
-% end
-
+func f(
+%for i in range(N):
+  collection${i}: [String],
+%end
+  collection${N+1}: [String]
+) {
   _ = ${' + '.join("collection%s" % i for i in range(N))}
 }

validation-test/Sema/type_checker_perf/fast/rdar26939465.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar29025667.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar30213053.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 3 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 3 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar30389602.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar30729643.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 15 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 15 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar31563957.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar32999041.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 3 --end 8 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 3 --end 8 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 // FIXME: https://bugs.swift.org/browse/SR-6997

validation-test/Sema/type_checker_perf/fast/rdar33292740.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 2 --end 10 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 2 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/fast/rdar40344044.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 3 --end 20  --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --begin 3 --end 20  --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/array_of_tuples.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 // REQUIRES: rdar38963783

validation-test/Sema/type_checker_perf/slow/nil_coalescing.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar20818064.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 3 --end 7 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 3 --end 7 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar20959612.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 1 --end 10 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar21198787.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 5 --end 20 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar24543332.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s --polynomial-threshold=1.7
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s --polynomial-threshold=1.7
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar27585838.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s --expected-exit-code 1
+// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select numLeafScopes %s --expected-exit-code 1
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar30596744_1.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select incrementScopeCounter %s --expected-exit-code 1
+// RUN: %scale-test --invert-result --begin 1 --end 5 --step 1 --select numLeafScopes %s --expected-exit-code 1
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar30596744_2.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --begin 1 --end 5 --step 1 --select incrementScopeCounter %s --polynomial-threshold 1.7
+// RUN: %scale-test --begin 1 --end 5 --step 1 --select numLeafScopes %s --polynomial-threshold 1.7
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 

validation-test/Sema/type_checker_perf/slow/rdar30606089.swift.gyb

@@ -1,4 +1,4 @@
-// RUN: %scale-test --invert-result --begin 2 --end 6 --step 1 --select incrementScopeCounter %s
+// RUN: %scale-test --invert-result --begin 2 --end 6 --step 1 --select numLeafScopes %s
 // REQUIRES: OS=macosx
 // REQUIRES: asserts
 // REQUIRES: rdar42969824