Sumanan's recent changes

doc/How to tune.txt

@@ -2,7 +2,7 @@ Go to lib/opentuner/ folder and run venv-bootstrap.py. This will setup a virtual
 
 Run the edu.mit.streamjit.tuner.ConfigGenerator.java to generate configuration information for a particular app. Tuner.java will update the apps table in the streamjit.db with name, configuration, location and class name of the streamjit app.
 
-Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder. If you need to change the input size, you can change it inside the function runApp() in the RunApp class.
+Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder.
 
 Run lib/opentuner/streamjit/tuner2.py.
 

jarapp.properties

@@ -0,0 +1,8 @@
+app=FMRadio
+streamGraphName=FMRadioCore
+#app=NestedSplitJoin
+#streamGraphName=NestedSplitJoinCore
+username=sumanan
+##password=abc123
+mainclasspath=edu.mit.streamjit.test.DistAppRunner
+numOfSNs=2

jarapp.xml

@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--Sumanan, sumanan@mit.edu, 15-12-2014 -->
+<project default="exportAll" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
+	<property file="jarapp.properties" />
+	<!--property name="app" value="NestedSplitJoin" /-->
+	<!--property name="mainclasspath" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->
+	<!-- If you want, you can hard code your username and the password as plain 
+		text here. Comment these properties (username or password or both) if you 
+		don't want to keep it as a plain text. If these properties are not set (commented), 
+		Ant will ask you to enter those at runtime. -->
+	<!-- property name="username" value="sumanan" /-->
+	<!-- property name="password" value="" / -->
+
+	<script language="javascript">
+		arr = project.getProperty('mainclasspath').split('.');
+		project.setProperty('mainclass', arr[arr.length -1]);
+	 </script>
+
+	<target name="exportApp.jar" description="Exports ${app}.jar">
+		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
+			<manifest>
+				<attribute name="Main-Class" value="${mainclasspath}" />
+				<attribute name="Class-Path" value="." />
+			</manifest>
+			<fileset dir="bin" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/jsonp/javax.json-1.0-fab.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ASM/asm-debug-all-4.1.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ServiceProviderProcessor/ServiceProviderProcessor.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/JOptSimple/jopt-simple-4.5.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/sqlite/sqlite-jdbc-3.7.2.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/Guava/guava-15.0.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar" />
+		</jar>
+	</target>
+
+	<target name="exportStreamNode.jar" description="Exports StreamNode.jar">
+		<jar destfile="StreamNode.jar" filesetmanifest="mergewithoutmain">
+			<manifest>
+				<attribute name="Main-Class" value="edu.mit.streamjit.impl.distributed.node.StreamNode" />
+				<attribute name="Class-Path" value="." />
+			</manifest>
+			<fileset dir="bin" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/jsonp/javax.json-1.0-fab.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ASM/asm-debug-all-4.1.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ServiceProviderProcessor/ServiceProviderProcessor.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/JOptSimple/jopt-simple-4.5.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/sqlite/sqlite-jdbc-3.7.2.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/Guava/guava-15.0.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar" />
+		</jar>
+	</target>
+
+	<target name="exportAll" depends="exportApp.jar, exportStreamNode.jar">
+	</target>
+
+	<target name="runStreamNode" description="Runs two instances of StreamNode.jar">
+		<exec dir="." executable="gnome-terminal" osfamily="unix">
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+		</exec>
+	</target>
+
+	<target name="runAll" description="Runs app.jar and two instances of StreamNode.jar">
+		<exec dir="." executable="gnome-terminal" osfamily="unix">
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar ${app}.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+		</exec>
+	</target>
+
+	<target name="exportAndRunAll" depends="exportAll, runAll">
+	</target>
+
+	<target name="sendtoLanka" depends="exportAll, getLogin">
+		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
+			<fileset file="StreamNode.jar" />
+			<fileset file="${app}.jar" />
+			<fileset file="optionsLanka.properties" />
+		</scp>
+	</target>
+
+	<target name="getLogin">
+		<input message="Enter Username:" addproperty="username" />
+		<input message="Enter Password:" addproperty="password">
+			<handler classname="org.apache.tools.ant.input.SecureInputHandler" />
+		</input>
+		<!--echo message="Username: ${username}" /-->
+		<!--echo message="Password: ${password}" /-->
+	</target>
+
+	<target name="remotesetup" depends="sendtoLanka, createDownloadScript">
+		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${streamGraphName} ${numOfSNs}" />
+	</target>
+
+	<!--http://www.vogella.com/tutorials/ApacheAnt/article.html-->
+	<!-- The value of a property is accessed via ${} -->
+	<property name="src.dir" location="src" />
+	<property name="lib.dir" location="lib" />
+	<property name="build.dir" location="bin" />
+
+	<!--Create a classpath container which can be later used in the ant task-->
+	<path id="build.classpath">
+		<fileset dir="${lib.dir}">
+			<include name="**/*.jar" />
+		</fileset>
+	</path>
+
+	<!-- Deletes the existing build directory-->
+	<target name="clean">
+		<delete dir="${build.dir}" />
+	</target>
+
+	<!-- Creates the  build  directory-->
+	<target name="makedir">
+		<mkdir dir="${build.dir}" />
+	</target>
+
+	<!-- Compiles the java code -->
+	<target name="compile" depends="clean, makedir">
+		<javac srcdir="${src.dir}" destdir="${build.dir}" classpathref="build.classpath" />
+	</target>
+
+	<target name="createDownloadScript">
+		<echo file="../Experiments/${app}.sh" append="false">#!/bin/bash
+#Author - Sumanan
+app=${app}
+mainClass=${streamGraphName}
+if [ -d $app ]; then
+	echo "$app exists. No downloads..."
+ 	exit
+fi
+mkdir -p $app
+cd $app
+mkdir -p $mainClass
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,*.orig,streamgraph.dot\} $mainClass/
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties\} .
+#to download everything.
+#rsync -avh --progress ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app .
+	</echo>
+	</target>
+</project>

lib/opentuner/streamjit/onlinetuner.py

@@ -17,21 +17,26 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, ss, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
-		self.sdk = ss
+		self.connection = connection
 		self.trycount = 0
+		self.config = configuration
 
 	def run(self, desired_result, input, limit):
 		self.trycount = self.trycount + 1
 		print self.trycount
-		cfg = desired_result.configuration.data
-		#self.niceprint(cfg)
-		self.sdk.sendmsg("%s\n"%cfg)
-		msg = self.sdk.recvmsg()
+
+		cfg_data = desired_result.configuration.data
+		#self.niceprint(cfg_data)
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+		self.connection.sendmsg(self.config.toJSON())
+
+		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
-			self.sdk.close()
+			self.connection.close()
 			sys.exit(1)
 		exetime = float(msg)
 		if exetime < 0:
@@ -55,31 +60,32 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = configuration.data
-		print "Final configuration", cfg
-		self.sdk.sendmsg("Completed")
-		self.sdk.sendmsg("%s\n"%cfg)
-		self.sdk.close()
+		cfg_data = configuration.data
+		print "Final configuration", cfg_data
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+
+		self.connection.sendmsg("Completed")
+		self.connection.sendmsg(self.config.toJSON())
+		self.connection.close()
 		sys.exit(0)
 
 
-def main(args, cfg, ss):
+def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = cfg.getAllParameters()
 	#print "\nFeature variables...."
-	for key in params.keys():
-		#print "\t", key
-  		manipulator.add_parameter(cfg.getParameter(key))
+	for p in cfg.getAllParameters().values():
+		manipulator.add_parameter(p)
 
-	mi = StreamJitMI(args, ss, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
 	m.main()
 
-def start(argv, cfg, ss):
+def start(argv, cfg, connection):
 	log = logging.getLogger(__name__)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
 
@@ -90,4 +96,4 @@ def start(argv, cfg, ss):
 	if not args.database:
     		args.database = 'sqlite:///' + args.program + '.db'
 
-	main(args, cfg, ss)
+	main(args, cfg, connection)

lib/opentuner/streamjit/sjparameters.py

@@ -1,7 +1,7 @@
 import deps #fix sys.path
 import random
 import opentuner
-from opentuner.search.manipulator import IntegerParameter, FloatParameter, SwitchParameter, PermutationParameter, ArrayParameter
+from opentuner.search.manipulator import IntegerParameter, FloatParameter, SwitchParameter, PermutationParameter, ParameterArray
 
 class sjIntegerParameter(IntegerParameter):
 	def __init__(self, name, min, max, value, javaClass = None, **kwargs):
@@ -86,7 +86,7 @@ def json_replacement(self):
 			"universe": self.universe,
 			"class": self.javaClass}
 
-class sjCompositionParameter(ArrayParameter):
+class sjCompositionParameter(ParameterArray):
 	def __init__(self, name, values, javaClass):
 		super(sjCompositionParameter, self).__init__(name, len(values), FloatParameter, 0.0, 1.0)
 		self.values = values