Some profiling infrastructure. With no small amount of fiddling

I avoided creating any dependency on yourkit.  In addition, there
was no way to give arguments to the JVM without losing the ones
defined in ANT_OPTS, which has been a massive pain for a while.
So there is now "jvm.opts" which is simply appended to ANT_OPTS, e.g.

  % ant -Djvm.opts=-verbose
  [echo] Forking with JVM opts: -Xms1536M -Xmx2g -Xss1M -XX:MaxPermSize=192M -XX:+UseParallelGC -verbose

There is a minimal stub defining a profiler interface:

  scala.tools.util.Profiling

Then the yourkit wrapper implements that interface.  Once your
locker has been rebuilt once, you can do this:

  ant yourkit.run

And it will build quick.lib/comp with profiling enabled, assuming
it can find the necessary files.  See the yourkit.init target for
values to change: or ant -Dyourkit.home=/path/to/it might be enough.

Review by dragos.

git-svn-id: http://lampsvn.epfl.ch/svn-repos/scala/scala/trunk@23528 5e8d7ff9-d8ef-0310-90f0-a4852d11357a

build.xml

@@ -1,7 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <project name="sabbus" default="build">
-
   <description>
 SuperSabbus for Scala core, builds the scala library and compiler. It can also package it as a simple distribution, tests it for stable bootstrapping and against the Scala test suite.
   </description>
@@ -185,16 +184,18 @@ PROPERTIES
   <!-- These are NOT the flags used to run SuperSabbus, but the ones written
        into the script runners created with scala.tools.ant.ScalaTool -->
   <property name="java.flags" value="-Xmx256M -Xms32M"/>
+  <property name="jvm.opts" value=""/>
 
   <!-- if ANT_OPTS is already set by the environment, it will be unaltered,
        but if it is unset it will take this default value. -->
   <property name="env.ANT_OPTS" value="-Xms1536M -Xmx1536M -Xss1M -XX:MaxPermSize=192M -XX:+UseParallelGC" />
+  
   <!--  to find max heap usage: -Xaprof ; currently at 980M for locker.comp -->
-  <echo message="Using ANT_OPTS: ${env.ANT_OPTS}" />
+  <echo message="Forking with JVM opts: ${env.ANT_OPTS} ${jvm.opts}" />
 
   <property
       name="scalacfork.jvmargs"
-      value="${env.ANT_OPTS}"/>
+      value="${env.ANT_OPTS} ${jvm.opts}"/>
 
 <!-- ===========================================================================
 INITIALISATION
@@ -255,7 +256,8 @@ INITIALISATION
   
     <!-- Local libs (developer use.) -->
     <mkdir dir="${lib-extra.dir}"/>
-    <path id="lib.dir.extra">
+    
+    <path id="lib.extra">
       <!-- needs ant 1.7.1 -->
       <!-- <fileset dir="${lib-extra.dir}" erroronmissingdir="false"> -->
       <fileset dir="${lib-extra.dir}">
@@ -271,7 +273,7 @@ INITIALISATION
         <include name="forkjoin.jar"/>
       </fileset>
       <pathelement location="${ant.jar}"/>
-      <path refid="lib.dir.extra"/>
+      <path refid="lib.extra"/>
     </path>
 
     <!-- Define tasks that can be run with Starr -->
@@ -284,12 +286,12 @@ INITIALISATION
     <path id="quick.compilation.path">
       <pathelement location="${build-quick.dir}/classes/library"/>
       <pathelement location="${lib.dir}/forkjoin.jar"/>
-      <path refid="lib.dir.extra"/>
+      <path refid="lib.extra"/>
     </path>
     <path id="strap.compilation.path">
       <pathelement location="${build-strap.dir}/classes/library"/>
       <pathelement location="${lib.dir}/forkjoin.jar"/>
-      <path refid="lib.dir.extra"/>
+      <path refid="lib.extra"/>
     </path>  
     <taskdef resource="scala/tools/ant/sabbus/antlib.xml" classpathref="starr.classpath"/>    
   </target>
@@ -872,6 +874,11 @@ PACKED QUICK BUILD (PACK)
       <zipfileset dirmode="755" filemode="644" src="${msil.jar}"/>
     </jar>
     <copy file="${jline.jar}" toDir="${build-pack.dir}/lib"/>
+    <copy todir="${build-pack.dir}/lib">
+      <fileset dir="${lib-extra.dir}">
+        <include name="**/*.jar"/>
+      </fileset>
+    </copy>
   </target>
   
   <target name="pack.pre-plugins" depends="pack.comp">
@@ -975,6 +982,7 @@ PACKED QUICK BUILD (PACK)
       <pathelement location="${build-pack.dir}/lib/scalap.jar"/>
       <pathelement location="${ant.jar}"/>
       <pathelement location="${jline.jar}"/>
+      <path refid="lib.extra"/>
     </path>
     <taskdef resource="scala/tools/ant/antlib.xml" classpathref="pack.classpath"/>
     <taskdef resource="scala/tools/partest/antlib.xml" classpathref="pack.classpath"/>
@@ -1907,6 +1915,50 @@ POSITIONS
 MISCELLANEOUS
 ============================================================================ -->
 
+  <target name="yourkit.init">
+    <property name="yourkit.home" value="/Applications/YourKit.app"/>
+    <property name="yourkit.api.jar" value="${yourkit.home}/lib/yjp-controller-api-redist.jar"/>
+    <property name="yourkit.agent" value="${yourkit.home}/bin/mac/libyjpagent.jnilib"/>
+    <property name="yourkit.jvm.opts" value="-agentpath:${yourkit.agent}"/>
+    <property name="yourkit.scalac.opts" value="-Yprofile:all"/>
+  </target>
+
+  <!-- Builds yourkit wrapper/jar and copies into lib/extra. -->
+  <target name="yourkit.build" depends="locker.done,yourkit.init">
+    <copy file="${yourkit.api.jar}" todir="${lib-extra.dir}"/>
+    <property name="yourkit.build.dir" value="${build-quick.dir}/classes/yourkit"/>
+    <mkdir dir="${yourkit.build.dir}"/>
+
+    <scalacfork
+      destdir="${yourkit.build.dir}"
+      compilerpathref="locker.classpath"
+      params="${scalac.args.all}"
+      srcdir="${src.dir}/yourkit"
+      jvmargs="${scalacfork.jvmargs}">
+      <include name="**/*.scala"/>
+      <compilationpath>
+        <path refid="locker.classpath"/>
+      </compilationpath>
+    </scalacfork>
+    <jar destfile="${lib-extra.dir}/scalac-yourkit.jar">
+      <fileset dir="${yourkit.build.dir}"/>
+    </jar>
+  </target>
+
+  <!-- Builds quick.lib/comp with profiling enabled. -->
+  <target name="yourkit.run" depends="yourkit.build">
+    <antcall target="clean"/>
+    <ant target="quick.lib" inheritall="false" inheritrefs="false">
+      <property name="jvm.opts" value="${yourkit.jvm.opts}"/>
+      <property name="scalac.args" value="${yourkit.scalac.opts}"/>
+    </ant>
+    <ant target="quick.comp" inheritall="false" inheritrefs="false">
+      <property name="jvm.opts" value="${yourkit.jvm.opts}"/>
+      <property name="scalac.args" value="${yourkit.scalac.opts}"/>
+    </ant>
+    <antcall target="build"/>
+  </target>
+
   <target name="graph.init">
     <taskdef name="vizant" classname="vizant.Vizant" classpath="${lib-ant.dir}/vizant.jar"/>
   </target>

src/compiler/scala/tools/nsc/Global.scala

@@ -9,6 +9,7 @@ import java.io.{ File, FileOutputStream, PrintWriter, IOException, FileNotFoundE
 import java.nio.charset.{ Charset, CharsetDecoder, IllegalCharsetNameException, UnsupportedCharsetException }
 import compat.Platform.currentTime
 
+import scala.tools.util.Profiling
 import scala.collection.{ mutable, immutable }
 import io.{ SourceReader, AbstractFile, Path }
 import reporters.{ Reporter, ConsoleReporter }
@@ -232,6 +233,7 @@ class Global(var settings: Settings, var reporter: Reporter) extends SymbolTable
     def logClasspath  = settings.Ylogcp.value
     def printLate     = settings.printLate.value
     def printStats    = settings.Ystatistics.value
+    def profileClass  = settings.YprofileClass.value
     def richExes      = settings.YrichExes.value
     def showTrees     = settings.Xshowtrees.value
     def target        = settings.target.value
@@ -247,9 +249,11 @@ class Global(var settings: Settings, var reporter: Reporter) extends SymbolTable
     def logPhase     = isActive(settings.log)
     def printPhase   = isActive(settings.Xprint)
     def showPhase    = isActive(settings.Yshow)
+    def profilePhase = isActive(settings.Yprofile) && !profileAll
 
     /** Derived values */
     def showNames     = List(showClass, showObject).flatten
+    def profileAll    = settings.Yprofile.doAllPhases
     def jvm           = target startsWith "jvm"
     def msil          = target == "msil"
     def verboseDebug  = debug && verbose
@@ -818,6 +822,9 @@ class Global(var settings: Settings, var reporter: Reporter) extends SymbolTable
     private def showMembers() =
       opt.showNames foreach (x => showDef(x, opt.declsOnly, globalPhase))
 
+    // If -Yprofile isn't given this will never be triggered.
+    lazy val profiler = Class.forName(opt.profileClass).newInstance().asInstanceOf[Profiling]
+
     /** Compile list of source files */
     def compileSources(_sources: List[SourceFile]) {
       val depSources = dependencyAnalysis.calculateFiles(_sources.distinct) // bug #1268, scalac confused by duplicated filenames
@@ -830,10 +837,20 @@ class Global(var settings: Settings, var reporter: Reporter) extends SymbolTable
       for (source <- sources) addUnit(new CompilationUnit(source))
 
       globalPhase = firstPhase
+
+      if (opt.profileAll) {
+        inform("starting CPU profiling on compilation run")
+        profiler.startProfiling()
+      }
       while (globalPhase != terminalPhase && !reporter.hasErrors) {
         val startTime = currentTime
         phase = globalPhase
-        globalPhase.run
+      
+        if (opt.profilePhase) {
+          inform("starting CPU profiling on phase " + globalPhase.name)
+          profiler profile globalPhase.run
+        }
+        else globalPhase.run
 
         // write icode to *.icode files
         if (opt.writeICode && (runIsAt(icodePhase) || opt.printPhase && runIsPast(icodePhase)))
@@ -866,6 +883,10 @@ class Global(var settings: Settings, var reporter: Reporter) extends SymbolTable
 
         advancePhase
       }
+      if (opt.profileAll) {
+        profiler.stopProfiling()
+        profiler.captureSnapshot()
+      }
   
       // If no phase was specified for -Xshow-class/object, show it now.
       if (settings.Yshow.isDefault)

src/compiler/scala/tools/nsc/io/package.scala

@@ -15,4 +15,13 @@ package object io {
   def runnableFn(f: () => Unit): Runnable     = runnable(f())
   def callableFn[T](f: () => T): Callable[T]  = callable(f())
   def spawnFn[T](f: () => T): Future[T]       = spawn(f())
+  
+  // Create, start, and return a background thread
+  // If isDaemon is true, it is marked as daemon (and will not interfere with JVM shutdown)
+  def daemonize(isDaemon: Boolean)(body: => Unit): Thread = {
+    val thread = new Thread(runnable(body))
+    thread setDaemon isDaemon
+    thread.start
+    thread
+  }  
 }

src/compiler/scala/tools/nsc/settings/ScalaSettings.scala

@@ -122,6 +122,8 @@ trait ScalaSettings extends AbsScalaSettings with StandardScalaSettings {
   val Ynogenericsig = BooleanSetting    ("-Yno-generic-signatures", "Suppress generation of generic signatures for Java")
   val noimports     = BooleanSetting    ("-Yno-imports", "Compile without any implicit imports")
   val nopredefs     = BooleanSetting    ("-Yno-predefs", "Compile without any implicit predefined values")
+  val Yprofile      = PhasesSetting     ("-Yprofile", "Profile the given phase. Needs yjpagent to run.")
+  val YprofileClass = StringSetting     ("-Yprofile-class", "class", "Name of profiler class", "scala.tools.util.YourkitProfiling")
   val Yrecursion    = IntSetting        ("-Yrecursion", "Recursion depth used when locking symbols", 0, Some(0, Int.MaxValue), (_: String) => None)
   val selfInAnnots  = BooleanSetting    ("-Yself-in-annots", "Include a \"self\" identifier inside of annotations")
   val Xshowtrees    = BooleanSetting    ("-Yshow-trees", "Show detailed trees when used in connection with -Xprint:<phase>")

src/compiler/scala/tools/util/Profiling.scala

@@ -0,0 +1,32 @@
+/* NSC -- new Scala compiler
+ * Copyright 2005-2010 LAMP/EPFL
+ * @author  Paul Phillips
+ */
+
+package scala.tools
+package util
+
+/** This is a (very) minimal stub for profiling, the purpose
+ *  of which is making it possible to integrate profiling hooks in
+ *  the compiler without creating a dependency on any particular
+ *  profiler.  You can specify a profiler class (which must be an
+ *  instance of this class) like so:
+ *  
+ *    // or -Yprofile:phase to profile individual phases
+ *    scalac -Yprofile-class your.profiler.Class -Yprofile:all <files>
+ *
+ */
+abstract class Profiling {
+  def isActive: Boolean
+  def startProfiling(): Unit
+  def stopProfiling(): Unit
+  def captureSnapshot(): Unit
+
+  def profile[T](body: => T): T = {
+    startProfiling()
+    val result = body
+    stopProfiling()
+    captureSnapshot()
+    result
+  }
+}

src/yourkit/scala/tools/util/YourkitProfiling.scala

@@ -0,0 +1,36 @@
+package scala.tools
+package util
+
+import com.yourkit.api._
+import nsc.io._
+
+class YourkitProfiling extends Profiling {
+  @volatile private var active = false
+  private var recordAllocation = false
+  lazy val controller = new Controller
+    
+  def startProfiling(): Unit = {
+    if (isActive)
+      return
+
+    active = true
+    daemonize(true) {
+      controller.startCPUProfiling(ProfilingModes.CPU_SAMPLING, Controller.DEFAULT_FILTERS)
+      if (recordAllocation)
+        controller.startAllocationRecording(true, 100, false, 0)
+    }
+  }
+
+  def captureSnapshot() =
+    daemonize(false)(controller.captureSnapshot(ProfilingModes.SNAPSHOT_WITH_HEAP))
+  
+  def stopProfiling() = {
+    if (recordAllocation)
+      controller.stopAllocationRecording()
+
+    controller.stopCPUProfiling()
+    active = false 
+  }
+  
+  def isActive = active
+}