From 1a7509558d6d711a088791ba3975f60c75133705 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 15:01:07 +0800
Subject: [PATCH 001/881] Meaningless variable name has been modified

ss -> connection
sdk -> connection
---
 lib/opentuner/streamjit/onlinetuner.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index d68c14f0..9b14ac66 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,9 +17,9 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, ss, manipulator, inputmanager, objective):
+	def __init__(self, args, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
-		self.sdk = ss
+		self.connection = connection
 		self.trycount = 0
 
 	def run(self, desired_result, input, limit):
@@ -27,11 +27,11 @@ def run(self, desired_result, input, limit):
 		print self.trycount
 		cfg = desired_result.configuration.data
 		#self.niceprint(cfg)
-		self.sdk.sendmsg("%s\n"%cfg)
-		msg = self.sdk.recvmsg()
+		self.connection.sendmsg("%s\n"%cfg)
+		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
-			self.sdk.close()
+			self.connection.close()
 			sys.exit(1)
 		exetime = float(msg)
 		if exetime < 0:
@@ -57,13 +57,13 @@ def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
 		cfg = configuration.data
 		print "Final configuration", cfg
-		self.sdk.sendmsg("Completed")
-		self.sdk.sendmsg("%s\n"%cfg)
-		self.sdk.close()
+		self.connection.sendmsg("Completed")
+		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.close()
 		sys.exit(0)
 
 
-def main(args, cfg, ss):
+def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
@@ -73,13 +73,13 @@ def main(args, cfg, ss):
 		#print "\t", key
   		manipulator.add_parameter(cfg.getParameter(key))
 	
-	mi = StreamJitMI(args, ss, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
 	m.main()
 
-def start(argv, cfg, ss):
+def start(argv, cfg, connection):
 	log = logging.getLogger(__name__)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
 
@@ -90,4 +90,4 @@ def start(argv, cfg, ss):
 	if not args.database:
     		args.database = 'sqlite:///' + args.program + '.db'
 
-	main(args, cfg, ss)
+	main(args, cfg, connection)

From e6f06589e8c015fe2909d905506ec31b4243a146 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:31:34 +0800
Subject: [PATCH 002/881] Receives Json string from python side of OpenTuner

Previously, python side open tuner was sending a python dictionary and it was
decoded and updated to the configuration object at Java side. Now python side
OT sends Json string.
---
 lib/opentuner/streamjit/onlinetuner.py        | 21 +++++++++++--------
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++--------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 9b14ac66..84e11b8e 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,17 +17,22 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, connection, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.connection = connection
 		self.trycount = 0
+		self.config = configuration
 
 	def run(self, desired_result, input, limit):
 		self.trycount = self.trycount + 1
 		print self.trycount
-		cfg = desired_result.configuration.data
-		#self.niceprint(cfg)
-		self.connection.sendmsg("%s\n"%cfg)
+
+		cfg_data = desired_result.configuration.data
+		#self.niceprint(cfg_data)
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+		self.connection.sendmsg(self.config.toJSON())
+
 		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
@@ -67,13 +72,11 @@ def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = cfg.getAllParameters()
 	#print "\nFeature variables...."
-	for key in params.keys():
-		#print "\t", key
-  		manipulator.add_parameter(cfg.getParameter(key))
+	for p in cfg.getAllParameters().values():
+		manipulator.add_parameter(p)
 	
-	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 55fff3e8..e1fc7910 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -27,7 +27,7 @@
 
 /**
  * Online tuner does continues learning.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Oct 8, 2013
  */
@@ -62,8 +62,8 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			String s = getConfigurationString(app.blobConfiguration);
-			tuner.writeLine(s);
+			// String s = getConfigurationString(app.blobConfiguration);
+			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
@@ -81,8 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				Configuration config = rebuildConfiguration(pythonDict,
-						app.blobConfiguration);
+				// Configuration config = rebuildConfiguration(pythonDict,
+				// app.blobConfiguration);
+
+				Configuration config = Configuration.fromJson(pythonDict);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -153,7 +155,7 @@ public void run() {
 
 	/**
 	 * Just excerpted from run() method for better readability.
-	 *
+	 * 
 	 * @throws IOException
 	 */
 	private void handleTermination() throws IOException {
@@ -174,7 +176,7 @@ private void handleTermination() throws IOException {
 	/**
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
-	 *
+	 * 
 	 * @param pythonDict
 	 */
 	private void runForever(String pythonDict) {
@@ -230,7 +232,7 @@ private void runForever(String pythonDict) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
@@ -279,7 +281,7 @@ private Configuration rebuildConfiguration(String pythonDict,
 	/**
 	 * TODO: This method is totally unnecessary if we remove the usage of the
 	 * name "class" in side {@link Configuration}.
-	 *
+	 * 
 	 * @param cfg
 	 * @return
 	 */

From e8df36aab35cac637b504db69a6799c5cdf22485 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:37:45 +0800
Subject: [PATCH 003/881] getConfigurationString() has been removed

It is an obselete method.
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e1fc7910..8b076932 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -278,21 +278,6 @@ private Configuration rebuildConfiguration(String pythonDict,
 		return builder.build();
 	}
 
-	/**
-	 * TODO: This method is totally unnecessary if we remove the usage of the
-	 * name "class" in side {@link Configuration}.
-	 * 
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From 971a03cd507c26bea660d1bb6a08dd9231d47755 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:41:29 +0800
Subject: [PATCH 004/881] Renaming: pythonDict -> cfgJson

---
 .../distributed/runtimer/OnlineTuner.java     | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8b076932..ef4a4a74 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -67,13 +67,13 @@ public void run() {
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
-				String pythonDict = tuner.readLine();
-				if (pythonDict == null)
+				String cfgJson = tuner.readLine();
+				if (cfgJson == null)
 					break;
 
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.
-				if (pythonDict.equals("Completed")) {
+				if (cfgJson.equals("Completed")) {
 					handleTermination();
 					break;
 				}
@@ -81,10 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(pythonDict,
+				// Configuration config = rebuildConfiguration(cfgJson,
 				// app.blobConfiguration);
 
-				Configuration config = Configuration.fromJson(pythonDict);
+				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -177,11 +177,11 @@ private void handleTermination() throws IOException {
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 */
-	private void runForever(String pythonDict) {
+	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(pythonDict,
+		Configuration config = rebuildConfiguration(cfgJson,
 				app.blobConfiguration);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
@@ -233,26 +233,26 @@ private void runForever(String pythonDict) {
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
 	 * @param config
 	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
+	 * @return New configuration object with updated values from the cfgJson.
 	 */
-	private Configuration rebuildConfiguration(String pythonDict,
+	private Configuration rebuildConfiguration(String cfgJson,
 			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
+		// System.out.println(cfgJson);
+		checkNotNull(cfgJson, "Received Python dictionary is null");
+		cfgJson = cfgJson.replaceAll("u'", "");
+		cfgJson = cfgJson.replaceAll("':", "");
+		cfgJson = cfgJson.replaceAll("\\{", "");
+		cfgJson = cfgJson.replaceAll("\\}", "");
 		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
 				.trimResults();
 		Configuration.Builder builder = Configuration.builder();
 		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
+		for (String s : dictSplitter.split(cfgJson)) {
 			String[] str = s.split(" ");
 			if (str.length != 2)
 				throw new AssertionError("Wrong python dictionary...");

From 774f61b69fe073bd7159f9686462326599d1379f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:46:40 +0800
Subject: [PATCH 005/881] rebuildConfiguration() has been removed

This is a obselete method
---
 .../distributed/runtimer/OnlineTuner.java     | 62 +------------------
 1 file changed, 3 insertions(+), 59 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ef4a4a74..8a4550cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -81,8 +81,6 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(cfgJson,
-				// app.blobConfiguration);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
@@ -161,7 +159,8 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		saveConfg(rebuildConfiguration(finalConfg, app.blobConfiguration), 0);
+		Configuration config = Configuration.fromJson(finalConfg);
+		saveConfg(config, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -181,8 +180,7 @@ private void handleTermination() throws IOException {
 	 */
 	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(cfgJson,
-				app.blobConfiguration);
+		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");
@@ -224,60 +222,6 @@ private void runForever(String cfgJson) {
 		}
 	}
 
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param cfgJson
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the cfgJson.
-	 */
-	private Configuration rebuildConfiguration(String cfgJson,
-			Configuration config) {
-		// System.out.println(cfgJson);
-		checkNotNull(cfgJson, "Received Python dictionary is null");
-		cfgJson = cfgJson.replaceAll("u'", "");
-		cfgJson = cfgJson.replaceAll("':", "");
-		cfgJson = cfgJson.replaceAll("\\{", "");
-		cfgJson = cfgJson.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(cfgJson)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-
-		}
-		return builder.build();
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From 574c265c7ba8f1c036103fb3fc3995ea004ac0d8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:49:21 +0800
Subject: [PATCH 006/881] saveConfg() directly takes configuration json...

string as an argument instead of configuration object.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8a4550cc..bc5ec943 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -85,7 +85,7 @@ public void run() {
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(config, tryCount);
+					saveConfg(cfgJson, tryCount);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {
@@ -159,8 +159,7 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		Configuration config = Configuration.fromJson(finalConfg);
-		saveConfg(config, 0);
+		saveConfg(finalConfg, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -225,8 +224,7 @@ private void runForever(String cfgJson) {
 	/**
 	 * Save the configuration.
 	 */
-	private void saveConfg(Configuration config, int round) {
-		String json = config.toJson();
+	private void saveConfg(String json, int round) {
 		try {
 
 			File dir = new File(String.format("configurations%s%s",

From 30cb237b24d73515de33d326a64cd20ac20e45c3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:15:49 +0800
Subject: [PATCH 007/881] Code cleanup

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index bc5ec943..a53e1a9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,21 +1,15 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Splitter;
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -62,7 +56,6 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			// String s = getConfigurationString(app.blobConfiguration);
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");

From 7d7b492801df53b6239c18c45d4641eb2cc24a57 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:37:14 +0800
Subject: [PATCH 008/881] Handles a type of termination condition

termination condition is input data finishes while running with fixed
configuration (i.e., no tuning, GlobalConstants.tune = false).
---
 .../impl/distributed/StreamJitAppManager.java |  1 -
 .../impl/distributed/TailChannel.java         | 34 ++++++++++++++++---
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index a213835e..27e24990 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -245,7 +245,6 @@ else if (GlobalConstants.useDrainData)
 
 		if (isFinal) {
 			this.status = AppStatus.STOPPED;
-			tailChannel.reset();
 			controller.closeAll();
 		}
 		isRunning = false;
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 36f8afc7..fa2e6c3e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Stopwatch;
 
@@ -21,6 +22,8 @@ public class TailChannel extends TCPInputChannel {
 
 	private volatile CountDownLatch latch;
 
+	private performanceLogger pLogger = null;
+
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
 			int limit) {
@@ -28,8 +31,10 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune)
-			new performanceLogger().start();
+		if (!GlobalConstants.tune) {
+			pLogger = new performanceLogger();
+			pLogger.start();
+		}
 	}
 
 	@Override
@@ -41,6 +46,15 @@ public void receiveData() {
 			latch.countDown();
 	}
 
+	@Override
+	public void stop(int type) {
+		super.stop(type);
+		if (pLogger != null) {
+			reset();
+			pLogger.stopLogging();
+		}
+	}
+
 	public void awaitForFixInput() throws InterruptedException {
 		latch.await();
 	}
@@ -53,6 +67,12 @@ public void reset() {
 
 	private class performanceLogger extends Thread {
 
+		private AtomicBoolean stopFlag;
+
+		private performanceLogger() {
+			stopFlag = new AtomicBoolean(false);
+		}
+
 		public void run() {
 			int i = 0;
 			FileWriter writer;
@@ -62,15 +82,15 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30) {
+			while (++i < 30 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();
 					stopwatch.stop();
 					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-					 System.out.println("Execution time is " + time
-					 + " milli seconds");
+					System.out.println("Execution time is " + time
+							+ " milli seconds");
 
 					writer.write(time.toString());
 					writer.write('\n');
@@ -88,5 +108,9 @@ public void run() {
 				e.printStackTrace();
 			}
 		}
+
+		public void stopLogging() {
+			stopFlag.set(true);
+		}
 	}
 }
\ No newline at end of file

From 2c1f342948a8d3aea6318dcb21dccac667b7dfcf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 23:49:31 +0800
Subject: [PATCH 009/881] OutputCount has been added to GlobalConstants.

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 +-
 .../streamjit/impl/distributed/common/GlobalConstants.java  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 27e24990..21348536 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, 1000);
+						+ tailToken.toString(), 0, GlobalConstants.outputCount);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 54f869a4..4215f21b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -99,6 +99,12 @@ private GlobalConstants() {
 	 */
 	public static final boolean saveAllConfigurations = true;
 
+	/**
+	 * Output count for tuning. Tuner measures the running time for this number
+	 * of outputs.
+	 */
+	public static final int outputCount = 100000;
+
 	static {
 
 	}

From d48ec5ede87c64e76a4d075190f418d7d54440c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 29 Jan 2014 23:24:44 +0800
Subject: [PATCH 010/881] Optional arg -> Positional arg

args.program is necessary
---
 lib/opentuner/streamjit/tuner3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner3.py b/lib/opentuner/streamjit/tuner3.py
index 9b21cf21..daa9e6ad 100755
--- a/lib/opentuner/streamjit/tuner3.py
+++ b/lib/opentuner/streamjit/tuner3.py
@@ -112,7 +112,7 @@ def make_jvm_options():
 if __name__ == '__main__':
 	logging.basicConfig(level=logging.INFO)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
-	parser.add_argument('--program', help='StreamJIT benchmark to tune (with first input)')
+	parser.add_argument('program', help='StreamJIT benchmark to tune (with first input)')
 	args = parser.parse_args()
 	(cfg_json, error_str) = call_java([], "edu.mit.streamjit.tuner.ConfigGenerator2",
 		["edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory", args.program])

From be81ef0d8ee53b4c0c5c64f2c86d0857c0fd1c03 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:43:25 +0800
Subject: [PATCH 011/881] Python dict is removed

Uses Json string to communicate with python process back and forth.
---
 lib/opentuner/streamjit/tuner2.py             | 40 +++++++-------
 .../mit/streamjit/tuner/ConfigGenerator.java  |  6 ++-
 src/edu/mit/streamjit/tuner/RunApp.java       | 52 ++++++-------------
 3 files changed, 41 insertions(+), 57 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 4b792432..97ffbf65 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -23,13 +23,14 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, jvmOptions, manipulator, inputmanager, objective):
 		args.technique = ['StreamJITBandit']
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.trycount = 0
 		self.jvmOptions = jvmOptions
 		self.program = args.program
 		self.StreamNodes = []
+		self.config = configuration
 		try:
 			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
 			c = self.tunedataDB.cursor()
@@ -43,15 +44,18 @@ def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
 			data = raw_input ( "Press Keyboard to exit..." )
 
 	def run(self, desired_result, input, limit):
-		cfg = dict.copy(desired_result.configuration.data)
-		(st, t) = self.runApp(cfg)
+		cfg_data = dict.copy(desired_result.configuration.data)
+		(st, t) = self.runApp(cfg_data)
 		return opentuner.resultsdb.models.Result(state=st, time=t)
 
 
-	def runApp(self, cfg):
+	def runApp(self, cfg_data):
 		self.trycount = self.trycount + 1
 		print '\n**********New Run - %d **********'%self.trycount
-		#self.niceprint(cfg)
+		#self.niceprint(cfg_data)
+
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
 
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
@@ -62,7 +66,7 @@ def runApp(self, cfg):
 		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
-			self.jvmOptions.get(key).setValue(cfg[key])
+			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
@@ -72,13 +76,12 @@ def runApp(self, cfg):
 		args.append(str(self.trycount))
 
 		cur = self.tunedataDB.cursor()
-		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, " ".join(jvmArgs), cfg, -1)
+		query = "INSERT INTO results VALUES (%d,'%s','%s', '%f')"%(self.trycount, " ".join(jvmArgs), self.config.toJSON(), -1)
 		cur.execute(query)
 		self.tunedataDB.commit()
-
 		p = subprocess.Popen(args, stderr=subprocess.PIPE)
-		if cfg.get('noOfMachines'):
-			self.startStreamNodes(cfg.get('noOfMachines') - 1, args)
+		if cfg_data.get('noOfMachines'):
+			self.startStreamNodes(cfg_data.get('noOfMachines') - 1, args)
 
 		timeout = 100
 
@@ -103,7 +106,7 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
 			str1 = str(commandStr)
-			str2 = str(cfg)
+			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()
 			return ('ERROR', float('inf'))
@@ -122,11 +125,11 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
-	def niceprint(self, cfg):
+	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount
-		for key in cfg.keys():
-			print "%s - %s"%(key, cfg[key])
+		for key in cfg_data.keys():
+			print "%s - %s"%(key, cfg_data[key])
 
 	def program_name(self):
 		return self.args.program
@@ -166,13 +169,15 @@ def main(args, cfg, jvmOptions):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = dict(cfg.items() + jvmOptions.items())
+	cfgparams = cfg.getAllParameters()
+
+	params = dict(cfgparams.items() + jvmOptions.items())
 	#print "\nFeature variables...."
 	for key in params.keys():
 		#print "\t", key
   		manipulator.add_parameter(params.get(key))
 	
-	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, jvmOptions, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
@@ -200,7 +205,6 @@ def start(program):
 			sys.exit(1)
 		cfgString = row[0]
 		cfg = configuration.getConfiguration(cfgString)
-		cfgparams = cfg.getAllParameters()
 	except Exception, e:
 		print 'Exception occured'
 		traceback.print_exc()
@@ -244,7 +248,7 @@ def start(program):
 	enabledJvmOptions = [aggressiveOpts, compileThreshold, freqInlineSize, maxInlineSize, maxInlineLevel]
 	jvmOptions = {x.name:x for x in enabledJvmOptions}
 
-	main(args, cfgparams, jvmOptions)
+	main(args, cfg, jvmOptions)
 
 if __name__ == '__main__':
 	prgrms = []
diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 3462e2d7..e1abd0de 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,6 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -102,7 +103,8 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		String confString = getConfigurationString(cfg);
+		// String confString = getConfigurationString(cfg);
+		String confString = cfg.toJson();
 
 		try {
 			sqlite.executeUpdate(String.format(
@@ -214,7 +216,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new Compiler2BlobFactory();
+			bf = new CompilerBlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index de342923..42a73049 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
@@ -27,10 +28,10 @@
  * information from streamjit.db based on the passed arguments, runs the
  * streamJit app and update the database with the execution time. StreamJit's
  * opentuner Python script calls this to run the streamJit application.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
- *
+ * 
  */
 public class RunApp {
 
@@ -50,62 +51,39 @@ public static void main(String[] args) throws SQLException {
 		System.out.println(String.format("JAVA Executing: %s Round - %d",
 				benchmarkName, round));
 
-		String dbPath = "streamjit.db";
-
-		sqliteAdapter sqlite;
-		try {
-			sqlite = new sqliteAdapter();
-		} catch (ClassNotFoundException e) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e.printStackTrace();
-			return;
-		}
-		sqlite.connectDB(dbPath);
-
-		ResultSet result = sqlite.executeQuery(String.format(
-				"SELECT * FROM apps WHERE name='%s'", benchmarkName));
-
-		String confgString = result.getString("configuration");
-
 		String sjDbPath = "sj" + benchmarkName + ".db";
 		sqliteAdapter sjDb;
 		try {
 			sjDb = new sqliteAdapter();
 		} catch (ClassNotFoundException e1) {
-			// Actually this exception will not occur. If Sqlite3 did not
-			// exists then it would have exit at previous return point.
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
 			e1.printStackTrace();
 			return;
 		}
-		sjDb.connectDB(sjDbPath);
 
-		ResultSet result1 = sjDb.executeQuery(String.format(
+		sjDb.connectDB(sjDbPath);
+		ResultSet result = sjDb.executeQuery(String.format(
 				"SELECT * FROM results WHERE Round=%d", round));
 
-		String pyDict = result1.getString("SJConfig");
-
-		Configuration config = Configuration
-				.fromJson(getConfigurationString(confgString));
-
-		Configuration cfg2 = rebuildConfiguration(pyDict, config);
+		String cfgJson = result.getString("SJConfig");
+		Configuration cfg = Configuration.fromJson(cfgJson);
 
 		Benchmark app = Benchmarker.getBenchmarkByName(benchmarkName);
 		StreamCompiler sc;
-		IntParameter p = cfg2.getParameter("noOfMachines", IntParameter.class);
+		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
-			csc.configuration(cfg2);
+			CompilerStreamCompiler csc = new CompilerStreamCompiler();
+			csc.setConfig(cfg);
 			sc = csc;
 		} else {
-			sc = new DistributedStreamCompiler(p.getValue(), cfg2);
+			sc = new DistributedStreamCompiler(p.getValue(), cfg);
 		}
 
 		double time = 0;
 		try {
-			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app, sc).get(0);
+			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app,
+					sc).get(0);
 			if (benchmarkResult.isOK())
 				time = benchmarkResult.runMillis();
 			else if (benchmarkResult.kind() == Benchmarker.Result.Kind.TIMEOUT)
@@ -118,7 +96,7 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				time = -2;
 			}
 		} catch (Exception e) {
-			//The Benchmarker should catch everything, but just in case...
+			// The Benchmarker should catch everything, but just in case...
 			e.printStackTrace();
 			time = -2;
 		} catch (OutOfMemoryError er) {
@@ -155,7 +133,7 @@ private static String getConfigurationString(String s) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.

From 570d9c0459dc9810ada86e21e4f076aaf929ae95 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:48:53 +0800
Subject: [PATCH 012/881] Code cleanup

Obselete methods have been removed. These method became obselete after the
last commit.
---
 .../mit/streamjit/tuner/ConfigGenerator.java  | 24 +------
 src/edu/mit/streamjit/tuner/RunApp.java       | 67 -------------------
 2 files changed, 3 insertions(+), 88 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index e1abd0de..414b2a50 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -25,7 +25,6 @@
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
-import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
  * ConfigGenerator generates {@link Configuration} of an application and stores
@@ -33,31 +32,16 @@
  * In this way, Opentuner can start and stop the StreamJit app for each tuning
  * try so that Opentuner can tune JVM parameters such as heapsize,
  * inlinethreshold, GCpausetime, etc as well.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
  */
 
 public class ConfigGenerator {
 
-	/**
-	 * TODO: Need to remove the string "class" from the {@link Configuration}
-	 * jsonifiers. Once it is done, this method can be removed.
-	 *
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Generates configuration for the passed provider.
-	 *
+	 * 
 	 * @param provider
 	 *            Only first benchmark is used to generate configuration. i.e.,
 	 *            only first benchmark will be tuned.
@@ -103,7 +87,6 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		// String confString = getConfigurationString(cfg);
 		String confString = cfg.toJson();
 
 		try {
@@ -141,14 +124,13 @@ public void connectDB(String path) {
 				statement.setQueryTimeout(30); // set timeout to 30 sec.
 
 			} catch (SQLException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
 
 		/**
 		 * Creates table iff it is not exists
-		 *
+		 * 
 		 * @param table
 		 *            Name of the table
 		 * @param signature
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 42a73049..38457a69 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -1,26 +1,18 @@
 package edu.mit.streamjit.tuner;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-import com.google.common.base.Splitter;
-
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
-import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
-import edu.mit.streamjit.test.Datasets;
 import edu.mit.streamjit.tuner.ConfigGenerator.sqliteAdapter;
 
 /**
@@ -117,63 +109,4 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				"UPDATE results SET Exectime=%f WHERE Round=%d", time, round);
 		sjDb.executeUpdate(qry);
 	}
-
-	private static String getConfigurationString(String s) {
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("javaClassPath", "class");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param pythonDict
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
-	 */
-	private static Configuration rebuildConfiguration(String pythonDict,
-			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-		}
-		return builder.build();
-	}
 }

From 64e2c6c2f7067f6dfb5a391da0bf443f18fc06e8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:59:34 +0800
Subject: [PATCH 013/881] Bug fix

commandStr -> jvmArgs
---
 lib/opentuner/streamjit/tuner2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 97ffbf65..239b086a 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -105,7 +105,7 @@ def runApp(self, cfg_data):
 			print "\033[31;1mException Found\033[0m"
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
-			str1 = str(commandStr)
+			str1 = str(jvmArgs)
 			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()

From 3d4fa6368ad2be53e9998092f4982e2e6580943a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:13:50 +0800
Subject: [PATCH 014/881] Saves Json of the final configuration

---
 lib/opentuner/streamjit/tuner2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 239b086a..376c9fa8 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -139,12 +139,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = dict.copy(configuration.data)
+		cfg_data = dict.copy(configuration.data)
+
 		print "\033[32;1mFinal Config...\033[0m"
-		(state, time) = self.runApp(cfg)
+		(state, time) = self.runApp(cfg_data)
 		conn = sqlite3.connect('streamjit.db', 100)
 		cur = conn.cursor()
-		query = 'INSERT INTO FinalResult VALUES ("%s","%s", %d, "%s", "%f")'%(self.program, cfg, self.trycount, state, float(time))
+		query = "INSERT INTO FinalResult VALUES ('%s','%s', %d, '%s', '%f')"%(self.program, self.config.toJSON(), self.trycount, state, float(time))
 		cur.execute(query)
 		conn.commit()
 

From 97b3696eedcb8dbcae45406d09e27c77ae6927ac Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:39:44 +0800
Subject: [PATCH 015/881] Uses compiler2

---
 src/edu/mit/streamjit/impl/interp/Interpreter.java | 12 ++++++------
 src/edu/mit/streamjit/tuner/ConfigGenerator.java   |  3 +--
 src/edu/mit/streamjit/tuner/RunApp.java            |  6 +++---
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/interp/Interpreter.java b/src/edu/mit/streamjit/impl/interp/Interpreter.java
index 609b86df..99c8fa88 100644
--- a/src/edu/mit/streamjit/impl/interp/Interpreter.java
+++ b/src/edu/mit/streamjit/impl/interp/Interpreter.java
@@ -115,9 +115,9 @@ public Interpreter(Iterable<Worker<?, ?>> workersIter, Iterable<MessageConstrain
 			Workers.getInputChannels(info.downstream()).set(info.getDownstreamChannelIndex(), channel);
 			if (initialState != null) {
 				ImmutableList<Object> data = initialState.getData(info.token());
-				System.out.println(String.format(
-						"Internal edge: InitialData of %s is %d", info.token(),
-						data.size()));
+//				System.out.println(String.format(
+//						"Internal edge: InitialData of %s is %d", info.token(),
+//						data.size()));
 				for (Object o : data != null ? data : ImmutableList.of())
 					channel.push(o);
 			}
@@ -151,9 +151,9 @@ public Interpreter(Iterable<Worker<?, ?>> workersIter, Iterable<MessageConstrain
 				//as input in the downstream blob.
 				if (initialState != null) {
 					ImmutableList<Object> data = initialState.getData(info.token());
-					System.out.println(String.format(
-							"External edge: InitialData of %s is %d",
-							info.token(), data.size()));
+//					System.out.println(String.format(
+//							"External edge: InitialData of %s is %d",
+//							info.token(), data.size()));
 					for (Object o : data != null ? data : ImmutableList.of())
 						channel.push(o);
 				}
diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 414b2a50..1d5d1401 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -198,7 +197,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new CompilerBlobFactory();
+			bf = new Compiler2BlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 38457a69..e0ca0db6 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -9,7 +9,7 @@
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
+import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
@@ -65,8 +65,8 @@ public static void main(String[] args) throws SQLException {
 		StreamCompiler sc;
 		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			CompilerStreamCompiler csc = new CompilerStreamCompiler();
-			csc.setConfig(cfg);
+			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
+			csc.configuration(cfg);
 			sc = csc;
 		} else {
 			sc = new DistributedStreamCompiler(p.getValue(), cfg);

From 63d19354dd7de520cee5a49c053e7879a06d54c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 09:58:52 +0800
Subject: [PATCH 016/881] Refactored

Methods getArgs1() and getArgs2() have been added.
---
 lib/opentuner/streamjit/tuner2.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 376c9fa8..63cf021e 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -60,18 +60,15 @@ def runApp(self, cfg_data):
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
 		javaPath = "java"
-		#the classpath, suitable as the value of the '-cp' java argument
-		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.6.2-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
 
-		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
 			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
-		args.extend(jvmArgs)
-		args.append("edu.mit.streamjit.tuner.RunApp")
+
+		args = self.getArgs1(javaPath, jvmArgs)
 		args.append(str(self.program))
 		args.append(str(self.trycount))
 
@@ -125,6 +122,22 @@ def runApp(self, cfg_data):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
+	# Return args that is to run a runnable jar file.
+	def getArgs1(self, javaPath, jvmArgs):
+		args = [javaPath]
+		args.extend(jvmArgs)
+		args.append("-jar")
+		args.append("RunApp.jar")
+		return args
+
+	# Return args that is to run from class file.
+	def getArgs2(self, javaPath, jvmArgs):
+		#the classpath, suitable as the value of the '-cp' java argument
+		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
+		args = [javaPath, "-cp", javaClassPath]
+		args.append("edu.mit.streamjit.tuner.RunApp")
+		return args
+
 	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount

From a435f7233f676d8e4717a328aabf646419bbbac1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 10:03:02 +0800
Subject: [PATCH 017/881] Updated to reflect latest changes

---
 doc/How to tune.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/How to tune.txt b/doc/How to tune.txt
index 34d0d0b6..ff247917 100644
--- a/doc/How to tune.txt	
+++ b/doc/How to tune.txt	
@@ -2,7 +2,7 @@ Go to lib/opentuner/ folder and run venv-bootstrap.py. This will setup a virtual
 
 Run the edu.mit.streamjit.tuner.ConfigGenerator.java to generate configuration information for a particular app. Tuner.java will update the apps table in the streamjit.db with name, configuration, location and class name of the streamjit app.
 
-Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder. If you need to change the input size, you can change it inside the function runApp() in the RunApp class.
+Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder.
 
 Run lib/opentuner/streamjit/tuner2.py.
 

From 9758c8261d7b2b7489bda0aa05dda9369c677293 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 10:17:36 +0800
Subject: [PATCH 018/881] More organized use of Configuration &  BlobFactory

StreamNodes uses the blobfactory which comes from Configuration.BlobSpecifier.
By fliping GlobalConstants.useCompilerBob flag, we can easily switch between
Interpreter blob and CompilerBlob.
---
 .../distributed/ConfigurationManager.java     | 35 ++++++++++++++++---
 .../distributed/DistributedBlobFactory.java   |  6 ++--
 .../distributed/common/GlobalConstants.java   |  5 +++
 .../node/CfgStringProcessorImpl.java          | 15 ++------
 .../impl/distributed/runtimer/Controller.java | 18 +---------
 5 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 21fe572f..67475cf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -19,9 +20,13 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
@@ -125,17 +130,21 @@ public Configuration getDynamicConfiguration() {
 			PartitionParameter.Builder partParam = PartitionParameter.builder(
 					GlobalConstants.PARTITION, coresPerMachine);
 
-			BlobFactory factory = new Interpreter.InterpreterBlobFactory();
-			partParam.addBlobFactory(factory);
-
+			BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
+			BlobFactory comp2Factory = new Compiler2BlobFactory();
+			partParam.addBlobFactory(intFactory);
+			partParam.addBlobFactory(comp2Factory);
 			app.blobtoMachineMap = new HashMap<>();
 
+			BlobFactory bf = GlobalConstants.useCompilerBlob
+					? comp2Factory
+					: intFactory;
 			for (Integer machineID : app.partitionsMachineMap.keySet()) {
 				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
 						.get(machineID);
 				for (Set<Worker<?, ?>> blobWorkers : blobList) {
 					// TODO: One core per blob. Need to change this.
-					partParam.addBlob(machineID, 1, factory, blobWorkers);
+					partParam.addBlob(machineID, 1, bf, blobWorkers);
 
 					// TODO: Temp fix to build.
 					Token t = Utils.getblobID(blobWorkers);
@@ -144,9 +153,25 @@ public Configuration getDynamicConfiguration() {
 			}
 
 			builder.addParameter(partParam.build());
-			if (app.blobConfiguration != null)
+			if (GlobalConstants.useCompilerBlob)
 				builder.addSubconfiguration("blobConfigs",
 						app.blobConfiguration);
+			else
+				builder.addSubconfiguration("blobConfigs",
+						getInterpreterConfg());
+			return builder.build();
+		}
+
+		private Configuration getInterpreterConfg() {
+			Configuration.Builder builder = Configuration.builder();
+			// TODO: Ensure the need of this switch parameter.
+			List<ChannelFactory> universe = Arrays
+					.<ChannelFactory> asList(new ConcurrentChannelFactory());
+			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
+					"channelFactory", ChannelFactory.class, universe.get(0),
+					universe);
+
+			builder.addParameter(cfParameter);
 			return builder.build();
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 51f14580..6c751363 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -10,6 +10,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
 /**
@@ -63,13 +64,14 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		Configuration distCfg = cfgManager.getDefaultConfiguration(workers,
 				noOfMachines);
-		Configuration.Builder builder = Configuration.builder(distCfg);
+		if (!GlobalConstants.useCompilerBlob)
+			return distCfg;
 
+		Configuration.Builder builder = Configuration.builder(distCfg);
 		BlobFactory compilerBf = new Compiler2BlobFactory();
 		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
-
 		return builder.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 4215f21b..ea7c44ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -105,6 +105,11 @@ private GlobalConstants() {
 	 */
 	public static final int outputCount = 100000;
 
+	/**
+	 * if true uses Compiler2, interpreter otherwise.
+	 */
+	public static final boolean useCompilerBlob = true;
+
 	static {
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index c937d94e..df029b2f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -25,14 +25,12 @@
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter.BlobSpecifier;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
-import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -128,32 +126,25 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			if (blobList == null)
 				return blobSet.build();
 
-			BlobFactory bf;
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
-			if (blobConfigs == null) {
-				blobConfigs = staticConfig;
-				bf = new Interpreter.InterpreterBlobFactory();
-			} else {
-				bf = new Compiler2BlobFactory();
-			}
 
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				// DEBUG
+				// DEBUG MSG
 				System.out.println(String.format(
 						"A new blob with workers %s has been created.",
 						workIdentifiers.toString()));
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 
 				try {
+					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;
 				}
 			}
-
 			return blobSet.build();
 		} else
 			return null;
@@ -200,7 +191,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 		URL url;
 		try {
 			url = jarFile.toURI().toURL();
-			URL[] urls = new URL[] { url };
+			URL[] urls = new URL[]{url};
 
 			ClassLoader loader = new URLClassLoader(urls);
 			Class<?> topStreamClass;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 7d42c667..84533dd1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -13,9 +12,7 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
@@ -29,7 +26,6 @@
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
-import edu.mit.streamjit.impl.interp.ChannelFactory;
 
 /**
  * {@link Controller} controls all {@link StreamNode}s in runtime. It has
@@ -126,25 +122,13 @@ public Map<Integer, Integer> getCoreCount() {
 
 	public void newApp(Configuration staticCfg) {
 		Configuration.Builder builder = Configuration.builder(staticCfg);
-
 		Map<Integer, InetAddress> inetMap = new HashMap<>();
 		for (StreamNodeAgent agent : StreamNodeMap.values())
 			inetMap.put(agent.getNodeID(), agent.getAddress());
 
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
-
-		// TODO: Ensure the need of this switch parameter.
-		List<ChannelFactory> universe = Arrays
-				.<ChannelFactory> asList(new ConcurrentChannelFactory());
-		SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
-				"channelFactory", ChannelFactory.class, universe.get(0),
-				universe);
-
-		builder.addParameter(cfParameter).putExtraData(
-				GlobalConstants.INETADDRESS_MAP, inetMap);
-
+		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
 		this.conProvider = new TCPConnectionProvider(controllerNodeID, inetMap);
-
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
 		sendToAll(json);

From 1a1fb2e326ac9278c761b7ef49273ff59d129a55 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 15:01:07 +0800
Subject: [PATCH 019/881] Meaningless variable name has been modified

ss -> connection
sdk -> connection
---
 lib/opentuner/streamjit/onlinetuner.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index d68c14f0..9b14ac66 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,9 +17,9 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, ss, manipulator, inputmanager, objective):
+	def __init__(self, args, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
-		self.sdk = ss
+		self.connection = connection
 		self.trycount = 0
 
 	def run(self, desired_result, input, limit):
@@ -27,11 +27,11 @@ def run(self, desired_result, input, limit):
 		print self.trycount
 		cfg = desired_result.configuration.data
 		#self.niceprint(cfg)
-		self.sdk.sendmsg("%s\n"%cfg)
-		msg = self.sdk.recvmsg()
+		self.connection.sendmsg("%s\n"%cfg)
+		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
-			self.sdk.close()
+			self.connection.close()
 			sys.exit(1)
 		exetime = float(msg)
 		if exetime < 0:
@@ -57,13 +57,13 @@ def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
 		cfg = configuration.data
 		print "Final configuration", cfg
-		self.sdk.sendmsg("Completed")
-		self.sdk.sendmsg("%s\n"%cfg)
-		self.sdk.close()
+		self.connection.sendmsg("Completed")
+		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.close()
 		sys.exit(0)
 
 
-def main(args, cfg, ss):
+def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
@@ -73,13 +73,13 @@ def main(args, cfg, ss):
 		#print "\t", key
   		manipulator.add_parameter(cfg.getParameter(key))
 	
-	mi = StreamJitMI(args, ss, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
 	m.main()
 
-def start(argv, cfg, ss):
+def start(argv, cfg, connection):
 	log = logging.getLogger(__name__)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
 
@@ -90,4 +90,4 @@ def start(argv, cfg, ss):
 	if not args.database:
     		args.database = 'sqlite:///' + args.program + '.db'
 
-	main(args, cfg, ss)
+	main(args, cfg, connection)

From bfa86ec10943a623b562bb65a200774551c5804a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:31:34 +0800
Subject: [PATCH 020/881] Receives Json string from python side of OpenTuner

Previously, python side open tuner was sending a python dictionary and it was
decoded and updated to the configuration object at Java side. Now python side
OT sends Json string.
---
 lib/opentuner/streamjit/onlinetuner.py        | 21 +++++++++++--------
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++--------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 9b14ac66..84e11b8e 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,17 +17,22 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, connection, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.connection = connection
 		self.trycount = 0
+		self.config = configuration
 
 	def run(self, desired_result, input, limit):
 		self.trycount = self.trycount + 1
 		print self.trycount
-		cfg = desired_result.configuration.data
-		#self.niceprint(cfg)
-		self.connection.sendmsg("%s\n"%cfg)
+
+		cfg_data = desired_result.configuration.data
+		#self.niceprint(cfg_data)
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+		self.connection.sendmsg(self.config.toJSON())
+
 		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
@@ -67,13 +72,11 @@ def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = cfg.getAllParameters()
 	#print "\nFeature variables...."
-	for key in params.keys():
-		#print "\t", key
-  		manipulator.add_parameter(cfg.getParameter(key))
+	for p in cfg.getAllParameters().values():
+		manipulator.add_parameter(p)
 	
-	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 55fff3e8..e1fc7910 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -27,7 +27,7 @@
 
 /**
  * Online tuner does continues learning.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Oct 8, 2013
  */
@@ -62,8 +62,8 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			String s = getConfigurationString(app.blobConfiguration);
-			tuner.writeLine(s);
+			// String s = getConfigurationString(app.blobConfiguration);
+			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
@@ -81,8 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				Configuration config = rebuildConfiguration(pythonDict,
-						app.blobConfiguration);
+				// Configuration config = rebuildConfiguration(pythonDict,
+				// app.blobConfiguration);
+
+				Configuration config = Configuration.fromJson(pythonDict);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -153,7 +155,7 @@ public void run() {
 
 	/**
 	 * Just excerpted from run() method for better readability.
-	 *
+	 * 
 	 * @throws IOException
 	 */
 	private void handleTermination() throws IOException {
@@ -174,7 +176,7 @@ private void handleTermination() throws IOException {
 	/**
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
-	 *
+	 * 
 	 * @param pythonDict
 	 */
 	private void runForever(String pythonDict) {
@@ -230,7 +232,7 @@ private void runForever(String pythonDict) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
@@ -279,7 +281,7 @@ private Configuration rebuildConfiguration(String pythonDict,
 	/**
 	 * TODO: This method is totally unnecessary if we remove the usage of the
 	 * name "class" in side {@link Configuration}.
-	 *
+	 * 
 	 * @param cfg
 	 * @return
 	 */

From c76550992d52a314c669d66bcd093a0d22ed3ce0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:37:45 +0800
Subject: [PATCH 021/881] getConfigurationString() has been removed

It is an obselete method.
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e1fc7910..8b076932 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -278,21 +278,6 @@ private Configuration rebuildConfiguration(String pythonDict,
 		return builder.build();
 	}
 
-	/**
-	 * TODO: This method is totally unnecessary if we remove the usage of the
-	 * name "class" in side {@link Configuration}.
-	 * 
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From c9de2f48dc8a1364cad160166dbb75e841ec7850 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:41:29 +0800
Subject: [PATCH 022/881] Renaming: pythonDict -> cfgJson

---
 .../distributed/runtimer/OnlineTuner.java     | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8b076932..ef4a4a74 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -67,13 +67,13 @@ public void run() {
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
-				String pythonDict = tuner.readLine();
-				if (pythonDict == null)
+				String cfgJson = tuner.readLine();
+				if (cfgJson == null)
 					break;
 
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.
-				if (pythonDict.equals("Completed")) {
+				if (cfgJson.equals("Completed")) {
 					handleTermination();
 					break;
 				}
@@ -81,10 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(pythonDict,
+				// Configuration config = rebuildConfiguration(cfgJson,
 				// app.blobConfiguration);
 
-				Configuration config = Configuration.fromJson(pythonDict);
+				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -177,11 +177,11 @@ private void handleTermination() throws IOException {
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 */
-	private void runForever(String pythonDict) {
+	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(pythonDict,
+		Configuration config = rebuildConfiguration(cfgJson,
 				app.blobConfiguration);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
@@ -233,26 +233,26 @@ private void runForever(String pythonDict) {
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
 	 * @param config
 	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
+	 * @return New configuration object with updated values from the cfgJson.
 	 */
-	private Configuration rebuildConfiguration(String pythonDict,
+	private Configuration rebuildConfiguration(String cfgJson,
 			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
+		// System.out.println(cfgJson);
+		checkNotNull(cfgJson, "Received Python dictionary is null");
+		cfgJson = cfgJson.replaceAll("u'", "");
+		cfgJson = cfgJson.replaceAll("':", "");
+		cfgJson = cfgJson.replaceAll("\\{", "");
+		cfgJson = cfgJson.replaceAll("\\}", "");
 		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
 				.trimResults();
 		Configuration.Builder builder = Configuration.builder();
 		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
+		for (String s : dictSplitter.split(cfgJson)) {
 			String[] str = s.split(" ");
 			if (str.length != 2)
 				throw new AssertionError("Wrong python dictionary...");

From a8830575ebf76f9965f65c33438ab669c645a7db Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:46:40 +0800
Subject: [PATCH 023/881] rebuildConfiguration() has been removed

This is a obselete method
---
 .../distributed/runtimer/OnlineTuner.java     | 62 +------------------
 1 file changed, 3 insertions(+), 59 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ef4a4a74..8a4550cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -81,8 +81,6 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(cfgJson,
-				// app.blobConfiguration);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
@@ -161,7 +159,8 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		saveConfg(rebuildConfiguration(finalConfg, app.blobConfiguration), 0);
+		Configuration config = Configuration.fromJson(finalConfg);
+		saveConfg(config, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -181,8 +180,7 @@ private void handleTermination() throws IOException {
 	 */
 	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(cfgJson,
-				app.blobConfiguration);
+		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");
@@ -224,60 +222,6 @@ private void runForever(String cfgJson) {
 		}
 	}
 
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param cfgJson
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the cfgJson.
-	 */
-	private Configuration rebuildConfiguration(String cfgJson,
-			Configuration config) {
-		// System.out.println(cfgJson);
-		checkNotNull(cfgJson, "Received Python dictionary is null");
-		cfgJson = cfgJson.replaceAll("u'", "");
-		cfgJson = cfgJson.replaceAll("':", "");
-		cfgJson = cfgJson.replaceAll("\\{", "");
-		cfgJson = cfgJson.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(cfgJson)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-
-		}
-		return builder.build();
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From c4f0f09098c53cc40969a4a630d85ad3f046c628 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:49:21 +0800
Subject: [PATCH 024/881] saveConfg() directly takes configuration json...

string as an argument instead of configuration object.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8a4550cc..bc5ec943 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -85,7 +85,7 @@ public void run() {
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(config, tryCount);
+					saveConfg(cfgJson, tryCount);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {
@@ -159,8 +159,7 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		Configuration config = Configuration.fromJson(finalConfg);
-		saveConfg(config, 0);
+		saveConfg(finalConfg, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -225,8 +224,7 @@ private void runForever(String cfgJson) {
 	/**
 	 * Save the configuration.
 	 */
-	private void saveConfg(Configuration config, int round) {
-		String json = config.toJson();
+	private void saveConfg(String json, int round) {
 		try {
 
 			File dir = new File(String.format("configurations%s%s",

From 9a6a199f2acbcda1c3b110d0fd26f5e815713086 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:15:49 +0800
Subject: [PATCH 025/881] Code cleanup

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index bc5ec943..a53e1a9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,21 +1,15 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Splitter;
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -62,7 +56,6 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			// String s = getConfigurationString(app.blobConfiguration);
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");

From 37ae1ffbcc9c47b3609d952091e085b52a802751 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:37:14 +0800
Subject: [PATCH 026/881] Handles a type of termination condition

termination condition is input data finishes while running with fixed
configuration (i.e., no tuning, GlobalConstants.tune = false).
---
 .../impl/distributed/StreamJitAppManager.java |  1 -
 .../impl/distributed/TailChannel.java         | 34 ++++++++++++++++---
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index a213835e..27e24990 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -245,7 +245,6 @@ else if (GlobalConstants.useDrainData)
 
 		if (isFinal) {
 			this.status = AppStatus.STOPPED;
-			tailChannel.reset();
 			controller.closeAll();
 		}
 		isRunning = false;
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 36f8afc7..fa2e6c3e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Stopwatch;
 
@@ -21,6 +22,8 @@ public class TailChannel extends TCPInputChannel {
 
 	private volatile CountDownLatch latch;
 
+	private performanceLogger pLogger = null;
+
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
 			int limit) {
@@ -28,8 +31,10 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune)
-			new performanceLogger().start();
+		if (!GlobalConstants.tune) {
+			pLogger = new performanceLogger();
+			pLogger.start();
+		}
 	}
 
 	@Override
@@ -41,6 +46,15 @@ public void receiveData() {
 			latch.countDown();
 	}
 
+	@Override
+	public void stop(int type) {
+		super.stop(type);
+		if (pLogger != null) {
+			reset();
+			pLogger.stopLogging();
+		}
+	}
+
 	public void awaitForFixInput() throws InterruptedException {
 		latch.await();
 	}
@@ -53,6 +67,12 @@ public void reset() {
 
 	private class performanceLogger extends Thread {
 
+		private AtomicBoolean stopFlag;
+
+		private performanceLogger() {
+			stopFlag = new AtomicBoolean(false);
+		}
+
 		public void run() {
 			int i = 0;
 			FileWriter writer;
@@ -62,15 +82,15 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30) {
+			while (++i < 30 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();
 					stopwatch.stop();
 					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-					 System.out.println("Execution time is " + time
-					 + " milli seconds");
+					System.out.println("Execution time is " + time
+							+ " milli seconds");
 
 					writer.write(time.toString());
 					writer.write('\n');
@@ -88,5 +108,9 @@ public void run() {
 				e.printStackTrace();
 			}
 		}
+
+		public void stopLogging() {
+			stopFlag.set(true);
+		}
 	}
 }
\ No newline at end of file

From d0248867771f325c2f5ea3b57b3ffd2aa225fd2f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 23:49:31 +0800
Subject: [PATCH 027/881] OutputCount has been added to GlobalConstants.

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 +-
 .../streamjit/impl/distributed/common/GlobalConstants.java  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 27e24990..21348536 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, 1000);
+						+ tailToken.toString(), 0, GlobalConstants.outputCount);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 54f869a4..4215f21b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -99,6 +99,12 @@ private GlobalConstants() {
 	 */
 	public static final boolean saveAllConfigurations = true;
 
+	/**
+	 * Output count for tuning. Tuner measures the running time for this number
+	 * of outputs.
+	 */
+	public static final int outputCount = 100000;
+
 	static {
 
 	}

From 54c684ffc11265bd2ae22c046d263d9a61a13bbb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 29 Jan 2014 23:24:44 +0800
Subject: [PATCH 028/881] Optional arg -> Positional arg

args.program is necessary
---
 lib/opentuner/streamjit/tuner3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner3.py b/lib/opentuner/streamjit/tuner3.py
index ce8ff3f9..2d5e53d9 100755
--- a/lib/opentuner/streamjit/tuner3.py
+++ b/lib/opentuner/streamjit/tuner3.py
@@ -120,7 +120,7 @@ def make_jvm_options():
 if __name__ == '__main__':
 	logging.basicConfig(level=logging.INFO)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
-	parser.add_argument('--program', help='StreamJIT benchmark to tune (with first input)')
+	parser.add_argument('program', help='StreamJIT benchmark to tune (with first input)')
 	args = parser.parse_args()
 	(cfg_json, error_str) = call_java([], "edu.mit.streamjit.tuner.ConfigGenerator2",
 		["edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory", args.program])

From f1fca23682262b582334629b567cf8384b732ab9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:43:25 +0800
Subject: [PATCH 029/881] Python dict is removed

Uses Json string to communicate with python process back and forth.
---
 lib/opentuner/streamjit/tuner2.py             | 40 +++++++-------
 .../mit/streamjit/tuner/ConfigGenerator.java  |  6 ++-
 src/edu/mit/streamjit/tuner/RunApp.java       | 52 ++++++-------------
 3 files changed, 41 insertions(+), 57 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 4b792432..97ffbf65 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -23,13 +23,14 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, jvmOptions, manipulator, inputmanager, objective):
 		args.technique = ['StreamJITBandit']
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.trycount = 0
 		self.jvmOptions = jvmOptions
 		self.program = args.program
 		self.StreamNodes = []
+		self.config = configuration
 		try:
 			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
 			c = self.tunedataDB.cursor()
@@ -43,15 +44,18 @@ def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
 			data = raw_input ( "Press Keyboard to exit..." )
 
 	def run(self, desired_result, input, limit):
-		cfg = dict.copy(desired_result.configuration.data)
-		(st, t) = self.runApp(cfg)
+		cfg_data = dict.copy(desired_result.configuration.data)
+		(st, t) = self.runApp(cfg_data)
 		return opentuner.resultsdb.models.Result(state=st, time=t)
 
 
-	def runApp(self, cfg):
+	def runApp(self, cfg_data):
 		self.trycount = self.trycount + 1
 		print '\n**********New Run - %d **********'%self.trycount
-		#self.niceprint(cfg)
+		#self.niceprint(cfg_data)
+
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
 
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
@@ -62,7 +66,7 @@ def runApp(self, cfg):
 		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
-			self.jvmOptions.get(key).setValue(cfg[key])
+			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
@@ -72,13 +76,12 @@ def runApp(self, cfg):
 		args.append(str(self.trycount))
 
 		cur = self.tunedataDB.cursor()
-		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, " ".join(jvmArgs), cfg, -1)
+		query = "INSERT INTO results VALUES (%d,'%s','%s', '%f')"%(self.trycount, " ".join(jvmArgs), self.config.toJSON(), -1)
 		cur.execute(query)
 		self.tunedataDB.commit()
-
 		p = subprocess.Popen(args, stderr=subprocess.PIPE)
-		if cfg.get('noOfMachines'):
-			self.startStreamNodes(cfg.get('noOfMachines') - 1, args)
+		if cfg_data.get('noOfMachines'):
+			self.startStreamNodes(cfg_data.get('noOfMachines') - 1, args)
 
 		timeout = 100
 
@@ -103,7 +106,7 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
 			str1 = str(commandStr)
-			str2 = str(cfg)
+			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()
 			return ('ERROR', float('inf'))
@@ -122,11 +125,11 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
-	def niceprint(self, cfg):
+	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount
-		for key in cfg.keys():
-			print "%s - %s"%(key, cfg[key])
+		for key in cfg_data.keys():
+			print "%s - %s"%(key, cfg_data[key])
 
 	def program_name(self):
 		return self.args.program
@@ -166,13 +169,15 @@ def main(args, cfg, jvmOptions):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = dict(cfg.items() + jvmOptions.items())
+	cfgparams = cfg.getAllParameters()
+
+	params = dict(cfgparams.items() + jvmOptions.items())
 	#print "\nFeature variables...."
 	for key in params.keys():
 		#print "\t", key
   		manipulator.add_parameter(params.get(key))
 	
-	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, jvmOptions, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
@@ -200,7 +205,6 @@ def start(program):
 			sys.exit(1)
 		cfgString = row[0]
 		cfg = configuration.getConfiguration(cfgString)
-		cfgparams = cfg.getAllParameters()
 	except Exception, e:
 		print 'Exception occured'
 		traceback.print_exc()
@@ -244,7 +248,7 @@ def start(program):
 	enabledJvmOptions = [aggressiveOpts, compileThreshold, freqInlineSize, maxInlineSize, maxInlineLevel]
 	jvmOptions = {x.name:x for x in enabledJvmOptions}
 
-	main(args, cfgparams, jvmOptions)
+	main(args, cfg, jvmOptions)
 
 if __name__ == '__main__':
 	prgrms = []
diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 3462e2d7..e1abd0de 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,6 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -102,7 +103,8 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		String confString = getConfigurationString(cfg);
+		// String confString = getConfigurationString(cfg);
+		String confString = cfg.toJson();
 
 		try {
 			sqlite.executeUpdate(String.format(
@@ -214,7 +216,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new Compiler2BlobFactory();
+			bf = new CompilerBlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index de342923..42a73049 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
@@ -27,10 +28,10 @@
  * information from streamjit.db based on the passed arguments, runs the
  * streamJit app and update the database with the execution time. StreamJit's
  * opentuner Python script calls this to run the streamJit application.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
- *
+ * 
  */
 public class RunApp {
 
@@ -50,62 +51,39 @@ public static void main(String[] args) throws SQLException {
 		System.out.println(String.format("JAVA Executing: %s Round - %d",
 				benchmarkName, round));
 
-		String dbPath = "streamjit.db";
-
-		sqliteAdapter sqlite;
-		try {
-			sqlite = new sqliteAdapter();
-		} catch (ClassNotFoundException e) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e.printStackTrace();
-			return;
-		}
-		sqlite.connectDB(dbPath);
-
-		ResultSet result = sqlite.executeQuery(String.format(
-				"SELECT * FROM apps WHERE name='%s'", benchmarkName));
-
-		String confgString = result.getString("configuration");
-
 		String sjDbPath = "sj" + benchmarkName + ".db";
 		sqliteAdapter sjDb;
 		try {
 			sjDb = new sqliteAdapter();
 		} catch (ClassNotFoundException e1) {
-			// Actually this exception will not occur. If Sqlite3 did not
-			// exists then it would have exit at previous return point.
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
 			e1.printStackTrace();
 			return;
 		}
-		sjDb.connectDB(sjDbPath);
 
-		ResultSet result1 = sjDb.executeQuery(String.format(
+		sjDb.connectDB(sjDbPath);
+		ResultSet result = sjDb.executeQuery(String.format(
 				"SELECT * FROM results WHERE Round=%d", round));
 
-		String pyDict = result1.getString("SJConfig");
-
-		Configuration config = Configuration
-				.fromJson(getConfigurationString(confgString));
-
-		Configuration cfg2 = rebuildConfiguration(pyDict, config);
+		String cfgJson = result.getString("SJConfig");
+		Configuration cfg = Configuration.fromJson(cfgJson);
 
 		Benchmark app = Benchmarker.getBenchmarkByName(benchmarkName);
 		StreamCompiler sc;
-		IntParameter p = cfg2.getParameter("noOfMachines", IntParameter.class);
+		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
-			csc.configuration(cfg2);
+			CompilerStreamCompiler csc = new CompilerStreamCompiler();
+			csc.setConfig(cfg);
 			sc = csc;
 		} else {
-			sc = new DistributedStreamCompiler(p.getValue(), cfg2);
+			sc = new DistributedStreamCompiler(p.getValue(), cfg);
 		}
 
 		double time = 0;
 		try {
-			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app, sc).get(0);
+			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app,
+					sc).get(0);
 			if (benchmarkResult.isOK())
 				time = benchmarkResult.runMillis();
 			else if (benchmarkResult.kind() == Benchmarker.Result.Kind.TIMEOUT)
@@ -118,7 +96,7 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				time = -2;
 			}
 		} catch (Exception e) {
-			//The Benchmarker should catch everything, but just in case...
+			// The Benchmarker should catch everything, but just in case...
 			e.printStackTrace();
 			time = -2;
 		} catch (OutOfMemoryError er) {
@@ -155,7 +133,7 @@ private static String getConfigurationString(String s) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.

From d870a3376a9d69334ff741194222221d7c5da43a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:48:53 +0800
Subject: [PATCH 030/881] Code cleanup

Obselete methods have been removed. These method became obselete after the
last commit.
---
 .../mit/streamjit/tuner/ConfigGenerator.java  | 24 +------
 src/edu/mit/streamjit/tuner/RunApp.java       | 67 -------------------
 2 files changed, 3 insertions(+), 88 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index e1abd0de..414b2a50 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -25,7 +25,6 @@
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
-import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
  * ConfigGenerator generates {@link Configuration} of an application and stores
@@ -33,31 +32,16 @@
  * In this way, Opentuner can start and stop the StreamJit app for each tuning
  * try so that Opentuner can tune JVM parameters such as heapsize,
  * inlinethreshold, GCpausetime, etc as well.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
  */
 
 public class ConfigGenerator {
 
-	/**
-	 * TODO: Need to remove the string "class" from the {@link Configuration}
-	 * jsonifiers. Once it is done, this method can be removed.
-	 *
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Generates configuration for the passed provider.
-	 *
+	 * 
 	 * @param provider
 	 *            Only first benchmark is used to generate configuration. i.e.,
 	 *            only first benchmark will be tuned.
@@ -103,7 +87,6 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		// String confString = getConfigurationString(cfg);
 		String confString = cfg.toJson();
 
 		try {
@@ -141,14 +124,13 @@ public void connectDB(String path) {
 				statement.setQueryTimeout(30); // set timeout to 30 sec.
 
 			} catch (SQLException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
 
 		/**
 		 * Creates table iff it is not exists
-		 *
+		 * 
 		 * @param table
 		 *            Name of the table
 		 * @param signature
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 42a73049..38457a69 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -1,26 +1,18 @@
 package edu.mit.streamjit.tuner;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-import com.google.common.base.Splitter;
-
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
-import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
-import edu.mit.streamjit.test.Datasets;
 import edu.mit.streamjit.tuner.ConfigGenerator.sqliteAdapter;
 
 /**
@@ -117,63 +109,4 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				"UPDATE results SET Exectime=%f WHERE Round=%d", time, round);
 		sjDb.executeUpdate(qry);
 	}
-
-	private static String getConfigurationString(String s) {
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("javaClassPath", "class");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param pythonDict
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
-	 */
-	private static Configuration rebuildConfiguration(String pythonDict,
-			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-		}
-		return builder.build();
-	}
 }

From e6cf2c8d30a88964a10e075b9b8242882685c092 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:59:34 +0800
Subject: [PATCH 031/881] Bug fix

commandStr -> jvmArgs
---
 lib/opentuner/streamjit/tuner2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 97ffbf65..239b086a 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -105,7 +105,7 @@ def runApp(self, cfg_data):
 			print "\033[31;1mException Found\033[0m"
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
-			str1 = str(commandStr)
+			str1 = str(jvmArgs)
 			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()

From e5142759253b050f60d56374dc5f3b08de5d864e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:13:50 +0800
Subject: [PATCH 032/881] Saves Json of the final configuration

---
 lib/opentuner/streamjit/tuner2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 239b086a..376c9fa8 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -139,12 +139,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = dict.copy(configuration.data)
+		cfg_data = dict.copy(configuration.data)
+
 		print "\033[32;1mFinal Config...\033[0m"
-		(state, time) = self.runApp(cfg)
+		(state, time) = self.runApp(cfg_data)
 		conn = sqlite3.connect('streamjit.db', 100)
 		cur = conn.cursor()
-		query = 'INSERT INTO FinalResult VALUES ("%s","%s", %d, "%s", "%f")'%(self.program, cfg, self.trycount, state, float(time))
+		query = "INSERT INTO FinalResult VALUES ('%s','%s', %d, '%s', '%f')"%(self.program, self.config.toJSON(), self.trycount, state, float(time))
 		cur.execute(query)
 		conn.commit()
 

From 408babf984385201f09e9da365b3d8edeb9ce253 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:39:44 +0800
Subject: [PATCH 033/881] Uses compiler2

---
 src/edu/mit/streamjit/tuner/ConfigGenerator.java | 3 +--
 src/edu/mit/streamjit/tuner/RunApp.java          | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 414b2a50..1d5d1401 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -198,7 +197,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new CompilerBlobFactory();
+			bf = new Compiler2BlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 38457a69..e0ca0db6 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -9,7 +9,7 @@
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
+import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
@@ -65,8 +65,8 @@ public static void main(String[] args) throws SQLException {
 		StreamCompiler sc;
 		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			CompilerStreamCompiler csc = new CompilerStreamCompiler();
-			csc.setConfig(cfg);
+			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
+			csc.configuration(cfg);
 			sc = csc;
 		} else {
 			sc = new DistributedStreamCompiler(p.getValue(), cfg);

From 14f0e162b316dd571f3205d97c793982f14960f3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 09:58:52 +0800
Subject: [PATCH 034/881] Refactored

Methods getArgs1() and getArgs2() have been added.
---
 lib/opentuner/streamjit/tuner2.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 376c9fa8..63cf021e 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -60,18 +60,15 @@ def runApp(self, cfg_data):
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
 		javaPath = "java"
-		#the classpath, suitable as the value of the '-cp' java argument
-		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.6.2-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
 
-		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
 			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
-		args.extend(jvmArgs)
-		args.append("edu.mit.streamjit.tuner.RunApp")
+
+		args = self.getArgs1(javaPath, jvmArgs)
 		args.append(str(self.program))
 		args.append(str(self.trycount))
 
@@ -125,6 +122,22 @@ def runApp(self, cfg_data):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
+	# Return args that is to run a runnable jar file.
+	def getArgs1(self, javaPath, jvmArgs):
+		args = [javaPath]
+		args.extend(jvmArgs)
+		args.append("-jar")
+		args.append("RunApp.jar")
+		return args
+
+	# Return args that is to run from class file.
+	def getArgs2(self, javaPath, jvmArgs):
+		#the classpath, suitable as the value of the '-cp' java argument
+		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
+		args = [javaPath, "-cp", javaClassPath]
+		args.append("edu.mit.streamjit.tuner.RunApp")
+		return args
+
 	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount

From 47a19ea362fe1b2d7034b22af211558699999ecf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 10:03:02 +0800
Subject: [PATCH 035/881] Updated to reflect latest changes

---
 doc/How to tune.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/How to tune.txt b/doc/How to tune.txt
index 34d0d0b6..ff247917 100644
--- a/doc/How to tune.txt	
+++ b/doc/How to tune.txt	
@@ -2,7 +2,7 @@ Go to lib/opentuner/ folder and run venv-bootstrap.py. This will setup a virtual
 
 Run the edu.mit.streamjit.tuner.ConfigGenerator.java to generate configuration information for a particular app. Tuner.java will update the apps table in the streamjit.db with name, configuration, location and class name of the streamjit app.
 
-Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder. If you need to change the input size, you can change it inside the function runApp() in the RunApp class.
+Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder.
 
 Run lib/opentuner/streamjit/tuner2.py.
 

From ddd7184e8329c95ce3b870bd07ad24012e05c38b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 10:17:36 +0800
Subject: [PATCH 036/881] More organized use of Configuration &  BlobFactory

StreamNodes uses the blobfactory which comes from Configuration.BlobSpecifier.
By fliping GlobalConstants.useCompilerBob flag, we can easily switch between
Interpreter blob and CompilerBlob.
---
 .../distributed/ConfigurationManager.java     | 35 ++++++++++++++++---
 .../distributed/DistributedBlobFactory.java   |  6 ++--
 .../distributed/common/GlobalConstants.java   |  5 +++
 .../node/CfgStringProcessorImpl.java          | 15 ++------
 .../impl/distributed/runtimer/Controller.java | 18 +---------
 5 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 21fe572f..67475cf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -19,9 +20,13 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
@@ -125,17 +130,21 @@ public Configuration getDynamicConfiguration() {
 			PartitionParameter.Builder partParam = PartitionParameter.builder(
 					GlobalConstants.PARTITION, coresPerMachine);
 
-			BlobFactory factory = new Interpreter.InterpreterBlobFactory();
-			partParam.addBlobFactory(factory);
-
+			BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
+			BlobFactory comp2Factory = new Compiler2BlobFactory();
+			partParam.addBlobFactory(intFactory);
+			partParam.addBlobFactory(comp2Factory);
 			app.blobtoMachineMap = new HashMap<>();
 
+			BlobFactory bf = GlobalConstants.useCompilerBlob
+					? comp2Factory
+					: intFactory;
 			for (Integer machineID : app.partitionsMachineMap.keySet()) {
 				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
 						.get(machineID);
 				for (Set<Worker<?, ?>> blobWorkers : blobList) {
 					// TODO: One core per blob. Need to change this.
-					partParam.addBlob(machineID, 1, factory, blobWorkers);
+					partParam.addBlob(machineID, 1, bf, blobWorkers);
 
 					// TODO: Temp fix to build.
 					Token t = Utils.getblobID(blobWorkers);
@@ -144,9 +153,25 @@ public Configuration getDynamicConfiguration() {
 			}
 
 			builder.addParameter(partParam.build());
-			if (app.blobConfiguration != null)
+			if (GlobalConstants.useCompilerBlob)
 				builder.addSubconfiguration("blobConfigs",
 						app.blobConfiguration);
+			else
+				builder.addSubconfiguration("blobConfigs",
+						getInterpreterConfg());
+			return builder.build();
+		}
+
+		private Configuration getInterpreterConfg() {
+			Configuration.Builder builder = Configuration.builder();
+			// TODO: Ensure the need of this switch parameter.
+			List<ChannelFactory> universe = Arrays
+					.<ChannelFactory> asList(new ConcurrentChannelFactory());
+			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
+					"channelFactory", ChannelFactory.class, universe.get(0),
+					universe);
+
+			builder.addParameter(cfParameter);
 			return builder.build();
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 51f14580..6c751363 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -10,6 +10,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
 /**
@@ -63,13 +64,14 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		Configuration distCfg = cfgManager.getDefaultConfiguration(workers,
 				noOfMachines);
-		Configuration.Builder builder = Configuration.builder(distCfg);
+		if (!GlobalConstants.useCompilerBlob)
+			return distCfg;
 
+		Configuration.Builder builder = Configuration.builder(distCfg);
 		BlobFactory compilerBf = new Compiler2BlobFactory();
 		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
-
 		return builder.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 4215f21b..ea7c44ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -105,6 +105,11 @@ private GlobalConstants() {
 	 */
 	public static final int outputCount = 100000;
 
+	/**
+	 * if true uses Compiler2, interpreter otherwise.
+	 */
+	public static final boolean useCompilerBlob = true;
+
 	static {
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index c937d94e..df029b2f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -25,14 +25,12 @@
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter.BlobSpecifier;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
-import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -128,32 +126,25 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			if (blobList == null)
 				return blobSet.build();
 
-			BlobFactory bf;
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
-			if (blobConfigs == null) {
-				blobConfigs = staticConfig;
-				bf = new Interpreter.InterpreterBlobFactory();
-			} else {
-				bf = new Compiler2BlobFactory();
-			}
 
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				// DEBUG
+				// DEBUG MSG
 				System.out.println(String.format(
 						"A new blob with workers %s has been created.",
 						workIdentifiers.toString()));
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 
 				try {
+					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;
 				}
 			}
-
 			return blobSet.build();
 		} else
 			return null;
@@ -200,7 +191,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 		URL url;
 		try {
 			url = jarFile.toURI().toURL();
-			URL[] urls = new URL[] { url };
+			URL[] urls = new URL[]{url};
 
 			ClassLoader loader = new URLClassLoader(urls);
 			Class<?> topStreamClass;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 7d42c667..84533dd1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -13,9 +12,7 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
@@ -29,7 +26,6 @@
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
-import edu.mit.streamjit.impl.interp.ChannelFactory;
 
 /**
  * {@link Controller} controls all {@link StreamNode}s in runtime. It has
@@ -126,25 +122,13 @@ public Map<Integer, Integer> getCoreCount() {
 
 	public void newApp(Configuration staticCfg) {
 		Configuration.Builder builder = Configuration.builder(staticCfg);
-
 		Map<Integer, InetAddress> inetMap = new HashMap<>();
 		for (StreamNodeAgent agent : StreamNodeMap.values())
 			inetMap.put(agent.getNodeID(), agent.getAddress());
 
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
-
-		// TODO: Ensure the need of this switch parameter.
-		List<ChannelFactory> universe = Arrays
-				.<ChannelFactory> asList(new ConcurrentChannelFactory());
-		SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
-				"channelFactory", ChannelFactory.class, universe.get(0),
-				universe);
-
-		builder.addParameter(cfParameter).putExtraData(
-				GlobalConstants.INETADDRESS_MAP, inetMap);
-
+		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
 		this.conProvider = new TCPConnectionProvider(controllerNodeID, inetMap);
-
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
 		sendToAll(json);

From 4be473e2bc6c8b8bd77627ca8015a3a2229377b4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 13:10:27 +0800
Subject: [PATCH 037/881] Sends json value of final confg

Final config was sent as a python dict. Now changed it to Json format.
I overlooked it.
---
 lib/opentuner/streamjit/onlinetuner.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 84e11b8e..115c971d 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -60,10 +60,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = configuration.data
-		print "Final configuration", cfg
+		cfg_data = configuration.data
+		print "Final configuration", cfg_data
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+
 		self.connection.sendmsg("Completed")
-		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.sendmsg(self.config.toJSON())
 		self.connection.close()
 		sys.exit(0)
 

From 11be67acddffa510f0f6be80dc99c0dd6660aef8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 17:01:06 +0800
Subject: [PATCH 038/881] Converts pythonDict to Configuration JSON

---
 .../impl/common/ConfigurationEditor.java      | 136 +++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index f615dcb9..d390ceef 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -1,30 +1,41 @@
 package edu.mit.streamjit.impl.common;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
 
+import com.google.common.base.Splitter;
+
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.BlobFactory;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.fmradio.FMRadio;
+import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
+import edu.mit.streamjit.util.json.Jsonifiers;
 
 public class ConfigurationEditor {
 
 	static String name;
 	static int noofwrks;
+
 	/**
 	 * @param args
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		edit(name, noofwrks);
+		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		 edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
+		// convert();
 	}
 
 	private static void generate(OneToOneElement<?, ?> stream) {
@@ -109,4 +120,127 @@ private static void print(String name) {
 			}
 		}
 	}
+
+	private static void convert() {
+		String appName = "ChannelVocoder7Kernel";
+		Configuration cfg = readConfiguration(String.format("%d%s", 1, appName));
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					String.format("%d%s.cfg", 0, appName)));
+			String pythonDict = reader.readLine();
+
+			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
+			saveConfg(finalCfg, 0);
+		} catch (Exception ex) {
+			ex.printStackTrace();
+		}
+	}
+
+	/**
+	 * Creates a new {@link Configuration} from the received python dictionary
+	 * string. This is not a good way to do.
+	 * <p>
+	 * TODO: Need to add a method to {@link Configuration} so that the
+	 * configuration object can be updated from the python dict string. Now we
+	 * are destructing the old confg object and recreating a new one every time.
+	 * Not a appreciatable way.
+	 * 
+	 * @param pythonDict
+	 *            Python dictionary string. Autotuner gives a dictionary of
+	 *            features with trial values.
+	 * @param config
+	 *            Old configuration object.
+	 * @return New configuration object with updated values from the pythonDict.
+	 */
+	private static Configuration rebuildConfiguration(String pythonDict,
+			Configuration config) {
+		// System.out.println(pythonDict);
+		pythonDict = pythonDict.replaceAll("u'", "");
+		pythonDict = pythonDict.replaceAll("':", "");
+		pythonDict = pythonDict.replaceAll("\\{", "");
+		pythonDict = pythonDict.replaceAll("\\}", "");
+		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
+				.trimResults();
+		Configuration.Builder builder = Configuration.builder();
+		System.out.println("New parameter values from Opentuner...");
+		for (String s : dictSplitter.split(pythonDict)) {
+			String[] str = s.split(" ");
+			if (str.length != 2)
+				throw new AssertionError("Wrong python dictionary...");
+			Parameter p = config.getParameter(str[0]);
+			if (p == null)
+				continue;
+			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
+			if (p instanceof IntParameter) {
+				IntParameter ip = (IntParameter) p;
+				builder.addParameter(new IntParameter(ip.getName(),
+						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
+
+			} else if (p instanceof SwitchParameter<?>) {
+				SwitchParameter sp = (SwitchParameter) p;
+				Class<?> type = sp.getGenericParameter();
+				int val = Integer.parseInt(str[1]);
+				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
+						type, sp.getUniverse().get(val), sp.getUniverse());
+				builder.addParameter(sp1);
+			}
+
+		}
+		return builder.build();
+	}
+
+	/**
+	 * TODO: This method is totally unnecessary if we remove the usage of the
+	 * name "class" in side {@link Configuration}.
+	 * 
+	 * @param cfg
+	 * @return
+	 */
+	private static String getConfigurationString(Configuration cfg) {
+		String s = Jsonifiers.toJson(cfg).toString();
+		String s1 = s.replaceAll("__class__", "ttttt");
+		String s2 = s1.replaceAll("class", "javaClassPath");
+		String s3 = s2.replaceAll("ttttt", "__class__");
+		return s3;
+	}
+
+	/**
+	 * Save the configuration.
+	 */
+	private static void saveConfg(Configuration config, int round) {
+		String json = config.toJson();
+		String name = "erer";
+		try {
+
+			File dir = new File(String.format("configurations%s%s",
+					File.separator, name));
+			if (!dir.exists())
+				if (!dir.mkdirs()) {
+					System.err.println("Make directory failed");
+					return;
+				}
+
+			File file = new File(dir, String.format("%d%s.cfg", round, name));
+			FileWriter writer = new FileWriter(file, false);
+			writer.write(json);
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static Configuration readConfiguration(String simpeName) {
+		String name = String.format("%s.cfg", simpeName);
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
 }

From 0655ce3952b7b0c1955e492eb284c0c84b1cb06a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 08:48:38 +0800
Subject: [PATCH 039/881] tryCount -> round

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index a53e1a9d..3fac55cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -46,7 +46,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
-		int tryCount = 0;
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -73,12 +73,12 @@ public void run() {
 
 				System.out
 						.println("----------------------------------------------");
-				System.out.println(tryCount++);
+				System.out.println(round++);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, tryCount);
+					saveConfg(cfgJson, round);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {

From 291415a1c7991d5ad9e83c5e1098d311e06b25d3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 09:20:46 +0800
Subject: [PATCH 040/881] Variable round has been made to global

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 3fac55cc..aefa10f8 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -32,6 +32,7 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
+	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -42,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
+		this.round = 0;
 	}
 
 	@Override
 	public void run() {
-		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,

From 11322e342e32e6c9a22d91f03643825e3399727b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:36:52 +0800
Subject: [PATCH 041/881] Refactored the run method.

New method reconfigure() has been created.
---
 .../distributed/runtimer/OnlineTuner.java     | 136 ++++++++++--------
 1 file changed, 75 insertions(+), 61 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index aefa10f8..f4c88a2b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
+import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -32,7 +33,6 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
-	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -43,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
-		this.round = 0;
 	}
 
 	@Override
 	public void run() {
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -59,6 +59,8 @@ public void run() {
 			tuner.writeLine("confg");
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
+			Pair<Boolean, Long> ret;
+
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
@@ -72,65 +74,12 @@ public void run() {
 					break;
 				}
 
-				System.out
-						.println("----------------------------------------------");
-				System.out.println(round++);
-
-				Configuration config = Configuration.fromJson(cfgJson);
-
-				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, round);
-
-				try {
-					if (!cfgManager.newConfiguration(config)) {
-						tuner.writeLine("-1");
-						continue;
-					}
-
-					if (manager.isRunning()) {
-						boolean state = drainer.startDraining(0);
-						if (!state) {
-							System.err
-									.println("Final drain has already been called. no more tuning.");
-							tuner.writeLine("exit");
-							break;
-						}
-
-						System.err.println("awaitDrainedIntrmdiate");
-						drainer.awaitDrainedIntrmdiate();
-
-						if (GlobalConstants.useDrainData) {
-							System.err.println("awaitDrainData...");
-							drainer.awaitDrainData();
-							DrainData drainData = drainer.getDrainData();
-							app.drainData = drainData;
-						}
-					}
-
-					drainer.setBlobGraph(app.blobGraph);
-					System.err.println("Reconfiguring...");
-					if (manager.reconfigure()) {
-						Stopwatch stopwatch = Stopwatch.createStarted();
-						manager.awaitForFixInput();
-						stopwatch.stop();
-						// TODO: need to check the manager's status before
-						// passing
-						// the time. Exceptions, final drain, etc may causes app
-						// to
-						// stop executing.
-						long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-
-						System.out.println("Execution time is " + time
-								+ " milli seconds");
-						tuner.writeLine(new Double(time).toString());
-					} else {
-						tuner.writeLine("-1");
-						continue;
-					}
-				} catch (Exception ex) {
-					System.err
-							.println("Couldn't compile the stream graph with this configuration");
-					tuner.writeLine("-1");
+				ret = reconfigure(cfgJson, round++);
+				if (ret.first) {
+					tuner.writeLine(new Double(ret.second).toString());
+				} else {
+					tuner.writeLine("exit");
+					break;
 				}
 			}
 
@@ -145,6 +94,71 @@ public void run() {
 		}
 	}
 
+	/**
+	 * @param cfgJson
+	 * @param round
+	 * @return if ret.first == false, then no more tuning. ret.second = running
+	 *         time in milliseconds.
+	 */
+	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+		long time;
+		System.out.println("----------------------------------------------");
+		System.out.println(round);
+		Configuration config = Configuration.fromJson(cfgJson);
+
+		if (GlobalConstants.saveAllConfigurations || round == 0)
+			saveConfg(cfgJson, round);
+
+		try {
+			if (!cfgManager.newConfiguration(config)) {
+				return new Pair<Boolean, Long>(true, -1l);
+			}
+
+			if (manager.isRunning()) {
+				boolean state = drainer.startDraining(0);
+				if (!state) {
+					System.err
+							.println("Final drain has already been called. no more tuning.");
+					return new Pair<Boolean, Long>(false, -1l);
+				}
+
+				System.err.println("awaitDrainedIntrmdiate");
+				drainer.awaitDrainedIntrmdiate();
+
+				if (GlobalConstants.useDrainData) {
+					System.err.println("awaitDrainData...");
+					drainer.awaitDrainData();
+					DrainData drainData = drainer.getDrainData();
+					app.drainData = drainData;
+				}
+			}
+
+			drainer.setBlobGraph(app.blobGraph);
+			System.err.println("Reconfiguring...");
+			if (manager.reconfigure()) {
+				Stopwatch stopwatch = Stopwatch.createStarted();
+				manager.awaitForFixInput();
+				stopwatch.stop();
+				// TODO: need to check the manager's status before
+				// passing
+				// the time. Exceptions, final drain, etc may causes app
+				// to
+				// stop executing.
+				time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+				System.out.println("Execution time is " + time
+						+ " milli seconds");
+			} else {
+				time = -1l;
+			}
+		} catch (Exception ex) {
+			System.err
+					.println("Couldn't compile the stream graph with this configuration");
+			time = -1l;
+		}
+		return new Pair<Boolean, Long>(true, time);
+	}
+
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 

From acf5622cb51673026ea2a309a1a0bd7054bfc015 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:58:16 +0800
Subject: [PATCH 042/881] reconfiguration() takes only confg as arg

---
 .../distributed/runtimer/OnlineTuner.java     | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f4c88a2b..9a29d767 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,6 +1,8 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
@@ -74,7 +76,15 @@ public void run() {
 					break;
 				}
 
-				ret = reconfigure(cfgJson, round++);
+				System.out
+						.println("----------------------------------------------");
+				System.out.println(round++);
+				Configuration config = Configuration.fromJson(cfgJson);
+
+				if (GlobalConstants.saveAllConfigurations)
+					saveConfg(cfgJson, round);
+
+				ret = reconfigure(config);
 				if (ret.first) {
 					tuner.writeLine(new Double(ret.second).toString());
 				} else {
@@ -100,15 +110,8 @@ public void run() {
 	 * @return if ret.first == false, then no more tuning. ret.second = running
 	 *         time in milliseconds.
 	 */
-	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+	private Pair<Boolean, Long> reconfigure(Configuration config) {
 		long time;
-		System.out.println("----------------------------------------------");
-		System.out.println(round);
-		Configuration config = Configuration.fromJson(cfgJson);
-
-		if (GlobalConstants.saveAllConfigurations || round == 0)
-			saveConfg(cfgJson, round);
-
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				return new Pair<Boolean, Long>(true, -1l);

From e77f290521662d8398f2c3653feebbf7823a84ee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:19:07 +0800
Subject: [PATCH 043/881] Evaluates the final configuration

---
 .../distributed/runtimer/OnlineTuner.java     | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9a29d767..d352c726 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -171,6 +171,14 @@ private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
+
+		Configuration cfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(cfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -232,6 +240,55 @@ private void runForever(String cfgJson) {
 		}
 	}
 
+	/**
+	 * Evaluates a configuration.
+	 * 
+	 * @param cfg
+	 *            configuration that needs to be evaluated
+	 * @param cfgName
+	 *            name of the configuration. This is just for logging purpose.
+	 */
+	private void evaluateConfig(Configuration cfg, String cfgName) {
+		FileWriter writer;
+		double total = 0;
+		int count = 3;
+		try {
+			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
+					true);
+			writer.write("\n----------------------------------------\n");
+			writer.write(String.format("Configuration name = %s\n", cfgName));
+			if (cfg != null) {
+				for (int i = 0; i < count; i++) {
+					Pair<Boolean, Long> ret = reconfigure(cfg);
+					writer.write(ret.second.toString());
+					writer.write('\n');
+					total += ret.second;
+				}
+				double avg = total / count;
+				writer.write(String.format("Average execution time = %f%n\n",
+						avg));
+			} else {
+				writer.write("Null configuration\n");
+			}
+			writer.close();
+		} catch (IOException e1) {
+			e1.printStackTrace();
+		}
+	}
+
+	private Configuration readConfiguration(String name) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
+
 	/**
 	 * Save the configuration.
 	 */

From dc5d424d1316b05edc83b839f65dc7d77913a438 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:22:41 +0800
Subject: [PATCH 044/881] runForever() takes configuration object

previously it took json string of the configuration
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d352c726..f0229b7b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -172,8 +172,8 @@ private void handleTermination() throws IOException {
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
 
-		Configuration cfg = Configuration.fromJson(finalConfg);
-		evaluateConfig(cfg, "Final configuration");
+		Configuration finalcfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
 				app.name));
@@ -186,19 +186,18 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalConfg);
+			runForever(finalcfg);
 		}
 	}
 
 	/**
-	 * TODO: Just copied from the run method. Code duplication between this
-	 * method and the run() method. Try to avoid duplicate code.
+	 * TODO: Just copied from the reconfigure method. Code duplication between
+	 * this method and the reconfigure() method. Try to avoid duplicate code.
 	 * 
-	 * @param cfgJson
+	 * @param config
 	 */
-	private void runForever(String cfgJson) {
+	private void runForever(Configuration config) {
 		System.out.println("runForever");
-		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");

From a04bae78d2718dc852e1fade6c9ee977fa64b52a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:36:14 +0800
Subject: [PATCH 045/881] Refactored

The method runForever() has been remove and its functionalities are covered
by reconfigure().
---
 .../distributed/runtimer/OnlineTuner.java     | 52 ++-----------------
 1 file changed, 4 insertions(+), 48 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f0229b7b..ed82eb5a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -186,56 +186,12 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalcfg);
-		}
-	}
-
-	/**
-	 * TODO: Just copied from the reconfigure method. Code duplication between
-	 * this method and the reconfigure() method. Try to avoid duplicate code.
-	 * 
-	 * @param config
-	 */
-	private void runForever(Configuration config) {
-		System.out.println("runForever");
-		try {
-			if (!cfgManager.newConfiguration(config)) {
-				System.err.println("Invalid final configuration.");
-				return;
-			}
-
-			if (manager.isRunning()) {
-				boolean state = drainer.startDraining(0);
-				if (!state) {
-					System.err
-							.println("Final drain has already been called. no more tuning.");
-					return;
-				}
-
-				System.err.println("awaitDrainedIntrmdiate");
-				drainer.awaitDrainedIntrmdiate();
-
-				if (GlobalConstants.useDrainData) {
-					System.err.println("awaitDrainData...");
-					drainer.awaitDrainData();
-					DrainData drainData = drainer.getDrainData();
-					app.drainData = drainData;
-				}
-
-				drainer.setBlobGraph(app.blobGraph);
-			}
-
-			System.err.println("Reconfiguring...");
-			boolean var = manager.reconfigure();
-			if (var) {
+			Pair<Boolean, Long> ret = reconfigure(finalcfg);
+			if (ret.first && ret.second > 0)
 				System.out
-						.println("Application is running with the final configuration.");
-			} else {
+						.println("Application is running forever with the final configuration.");
+			else
 				System.err.println("Invalid final configuration.");
-			}
-		} catch (Exception ex) {
-			System.err
-					.println("Couldn't compile the stream graph with this configuration");
 		}
 	}
 

From 42e24b7bba586a86ddd32a7347054adc019ec373 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 18:53:27 +0800
Subject: [PATCH 046/881] Comments added

---
 .../streamjit/impl/distributed/ConfigurationManager.java   | 7 +++----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java    | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 67475cf3..b4ab69c6 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -34,14 +34,14 @@
  * ConfigurationManager deals with {@link Configuration}. Mainly, It does
  * following two tasks.
  * <ol>
- * <li>Generates configuration for with appropriate tuning parameters for
- * tuning.
+ * <li>Generates configuration with appropriate tuning parameters (Based on the
+ * search space design strategy) for tuning.
  * <li>Dispatch the configuration given by the open tuner and make blobs
  * accordingly.
  * </ol>
  * 
  * One can implement this interface to try different search space designs as
- * they want.
+ * he want.
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Jan 16, 2014
@@ -164,7 +164,6 @@ public Configuration getDynamicConfiguration() {
 
 		private Configuration getInterpreterConfg() {
 			Configuration.Builder builder = Configuration.builder();
-			// TODO: Ensure the need of this switch parameter.
 			List<ChannelFactory> universe = Arrays
 					.<ChannelFactory> asList(new ConcurrentChannelFactory());
 			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index fa2e6c3e..c7e21c94 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -82,7 +82,7 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30 && !stopFlag.get()) {
+			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();

From 2b17428f85029b61d48de51e3281cc48f3b4ac3f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:33:52 +0800
Subject: [PATCH 047/881] Prints status msg

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ed82eb5a..4374b466 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -204,6 +204,7 @@ private void handleTermination() throws IOException {
 	 *            name of the configuration. This is just for logging purpose.
 	 */
 	private void evaluateConfig(Configuration cfg, String cfgName) {
+		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
 		int count = 3;

From 8c36630bcb5dacfa244125132fed93992eecd084 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:34:40 +0800
Subject: [PATCH 048/881] Print msgs have been commented out

---
 .../impl/distributed/HotSpotTuning.java       | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 4b8a6154..deff1201 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -44,16 +44,16 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	@Override
 	public boolean newConfiguration(Configuration config) {
 
-		for (Parameter p : config.getParametersMap().values()) {
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				System.out.println(ip.getName() + " - " + ip.getValue());
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter<?> sp = (SwitchParameter<?>) p;
-				System.out.println(sp.getName() + " - " + sp.getValue());
-			} else
-				System.out.println(p.getName() + " - Unknown type");
-		}
+		// for (Parameter p : config.getParametersMap().values()) {
+		// if (p instanceof IntParameter) {
+		// IntParameter ip = (IntParameter) p;
+		// System.out.println(ip.getName() + " - " + ip.getValue());
+		// } else if (p instanceof SwitchParameter<?>) {
+		// SwitchParameter<?> sp = (SwitchParameter<?>) p;
+		// System.out.println(sp.getName() + " - " + sp.getValue());
+		// } else
+		// System.out.println(p.getName() + " - Unknown type");
+		// }
 
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
 		try {
@@ -165,11 +165,10 @@ private class PickHotSpots extends StreamVisitor {
 
 		private Joiner<?, ?> skipJoiner;
 
-		private int minSplitjoinSize = 20;
+		private int minSplitjoinSize = 8;
 
 		/**
-		 * Workers those are going to be part {@link OfflineTuner}
-		 * {@link #currentHotSpot}.
+		 * Workers those are going to be part of {@link #currentHotSpot}.
 		 */
 		List<Worker<?, ?>> workerGropups;
 

From feada6e67eb9eabc5c86ca5045ac0d21f128fb6c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:13:48 +0800
Subject: [PATCH 049/881] Comments added

---
 .../streamjit/impl/common/AbstractDrainer.java  | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index c7ae3634..35abf078 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -137,15 +137,14 @@ public final void setBlobGraph(BlobGraph blobGraph) {
 	 * Initiate the draining of the blobgraph. Three type of draining could be
 	 * carried out.
 	 * <ol>
-	 * <li>type 0 - Intermediate draining: In this case, no data from input
-	 * buffer will be consumed and StreamJit app will not be stopped. Rather,
-	 * StreamJit app will be just paused for reconfiguration purpose. This
-	 * draining may be triggered by {@link OnlineTuner}.</li>
-	 * <li>type 1 - Semi final draining: In this case, no data from input buffer
-	 * will be consumed but StreamJit app will be stopped. i.e, StreamJit app
-	 * will be stopped safely without consuming any new input. This draining may
-	 * be triggered by {@link OnlineTuner} after opentuner finish tuning and
-	 * send it's final configuration.</li>
+	 * <li>type 0 - Intermediate draining: In this case, no new data from
+	 * {@link Input} will be consumed and StreamJit app will not be stopped.
+	 * Rather, StreamJit app will be just paused for reconfiguration purpose.
+	 * This draining may be triggered by {@link OnlineTuner}.</li>
+	 * <li>type 1 - Semi final draining: In this case, StreamJit app will be
+	 * stopped safely without consuming any new data from {@link Input}. This
+	 * draining may be triggered by {@link OnlineTuner} after opentuner finish
+	 * tuning and send it's final configuration.</li>
 	 * <li>type 2 - Final draining: At the end of input data. After this
 	 * draining StreamJit app will stop. This draining may be triggered by a
 	 * {@link Input} when it run out of input data.</li>

From 8600c5b866b2213f2f99d01d338d1c3ed71cdca4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:15:34 +0800
Subject: [PATCH 050/881] Format changes

---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index d390ceef..2830d72f 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -32,8 +32,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		 edit(name, noofwrks);
+		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}

From e3d9f8a1338e6f3471b271a80cd90318ecb718d8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:16:28 +0800
Subject: [PATCH 051/881] Deadlock situation at final drain has been handled

Whenever GlobalConstants.useDrainData is false, irrespective of reqDrainData,
stop(3) will be called. This avoids deadlock.
---
 .../impl/distributed/node/BlobsManagerImpl.java      | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2f3ceca8..37d72161 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -298,10 +298,14 @@ private void doDrain(boolean reqDrainData) {
 			drainState = 1;
 
 			for (BoundaryInputChannel bc : inputChannels.values()) {
-				if (!this.reqDrainData)
-					bc.stop(1);
-				else if (GlobalConstants.useDrainData)
-					bc.stop(2);
+				// TODO: [2014-02-05] rearranged this order to call stop(3)
+				// whenever GlobalConstants.useDrainData is false irrespective
+				// of reqDrainData.
+				if (GlobalConstants.useDrainData)
+					if (!this.reqDrainData)
+						bc.stop(1);
+					else
+						bc.stop(2);
 				else
 					bc.stop(3);
 			}

From 4093bda9b2149addb6110448301944ea1cd57061 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 22:16:07 +0800
Subject: [PATCH 052/881] Filename changed

"hand_" prefix has been added to hand tuned configuration file generation.
---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 2830d72f..4f6005f6 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -51,7 +51,7 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("%s.cfg", stream.getClass().getSimpleName());
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
 
 		try {
 			FileWriter writer = new FileWriter(name, false);

From 0645d832093d18cb2779951218e187b3f7e27074 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 6 Feb 2014 00:02:16 +0800
Subject: [PATCH 053/881] New methods to deal with HotSpotTuning searchspace

Generates and edits the configuration files those are created by
HotSpotTuning ConfigurationManager.
---
 .../impl/common/ConfigurationEditor.java      | 90 ++++++++++++++++++-
 1 file changed, 87 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 4f6005f6..52518ccd 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,8 +16,12 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.impl.distributed.HotSpotTuning;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
 import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
 import edu.mit.streamjit.util.json.Jsonifiers;
@@ -32,8 +36,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		edit(name, noofwrks);
+		generate1(new FilterBank6.FilterBankPipeline());
+		edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}
@@ -61,7 +65,6 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
-
 	}
 
 	private static void edit(String name, int maxWor)
@@ -99,6 +102,87 @@ private static void edit(String name, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
+	private static void generate1(OneToOneElement<?, ?> stream) {
+		int noOfnodes = 4;
+
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
+		Worker<?, ?> sink = (Worker<?, ?>) primitiveConnector.getSink();
+		noofwrks = Workers.getIdentifier(sink) + 1;
+
+		StreamJitApp app = new StreamJitApp(stream, source, sink);
+		ConfigurationManager cfgManager = new HotSpotTuning(app);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, noOfnodes);
+
+		Configuration cfg = bf.getDefaultConfiguration(Workers
+				.getAllWorkersInGraph(source));
+
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+
+		try {
+			FileWriter writer = new FileWriter(name, false);
+			writer.write(cfg.toJson());
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static void edit1(String name, int maxWor)
+			throws NumberFormatException, IOException {
+		Configuration cfg;
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			cfg = Configuration.fromJson(json);
+			reader.close();
+		} catch (Exception ex) {
+			System.err.println("File reader error");
+			return;
+		}
+
+		Configuration.Builder builder = Configuration.builder(cfg);
+		BufferedReader keyinreader = new BufferedReader(new InputStreamReader(
+				System.in));
+
+		for (int i = 0; i < maxWor; i++) {
+			String wrkrMachineName = String.format("worker%dtomachine", i);
+			String wrkrCutname = String.format("worker%dcut", i);
+
+			SwitchParameter<Integer> wrkrMachine = cfg.getParameter(
+					wrkrMachineName, SwitchParameter.class);
+			IntParameter wrkrCut = cfg.getParameter(wrkrCutname,
+					IntParameter.class);
+
+			if (wrkrMachine != null) {
+				System.out.println(wrkrMachine.getName() + " - "
+						+ wrkrMachine.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrMachine.getName());
+				builder.addParameter(new SwitchParameter<Integer>(wrkrMachine
+						.getName(), Integer.class, val, wrkrMachine
+						.getUniverse()));
+			}
+
+			if (wrkrCut != null) {
+				System.out.println(wrkrCut.getName() + " - "
+						+ wrkrCut.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrCut.getName());
+				builder.addParameter(new IntParameter(wrkrCut.getName(),
+						wrkrCut.getRange(), val));
+			}
+		}
+
+		cfg = builder.build();
+		FileWriter writer = new FileWriter(name);
+		writer.write(cfg.toJson());
+		writer.close();
+		System.out.println("Successfully updated");
+	}
+
 	private static void print(String name) {
 		Configuration cfg;
 		try {

From 34330f7e72804c38fa3ff55fd1c0fd316749a533 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:12:54 +0800
Subject: [PATCH 054/881] GlobalConstants.tune made to int

Third running mode, evaluates two configuration files mode, has been added.
Previous two are tune and uses cfg file.
---
 .../impl/distributed/DistributedStreamCompiler.java      | 7 ++-----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java  | 2 +-
 .../impl/distributed/common/GlobalConstants.java         | 9 ++++++++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 25f3e799..d72c22aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,11 +129,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune) {
-
-		} else {
+		if (GlobalConstants.tune == 0)
 			this.cfg = readConfiguration(stream.getClass().getSimpleName());
-		}
 
 		if (cfg == null) {
 			System.err
@@ -200,7 +197,7 @@ public void drain() {
 		manager.reconfigure();
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
-		if (GlobalConstants.tune && this.cfg != null) {
+		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
 					cfgManager, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index c7e21c94..13339177 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -31,7 +31,7 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune) {
+		if (GlobalConstants.tune == 0) {
 			pLogger = new performanceLogger();
 			pLogger.start();
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index ea7c44ac..ffd35ad5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -90,8 +90,15 @@ private GlobalConstants() {
 	 * to pass fixed number of input will be measured for 30 rounds and logged
 	 * into FixedOutPut.txt. See {@link TailChannel} for the file logging
 	 * details.
+	 * <ol>
+	 * 0 - No tuning, uses configuration file to run.
+	 * <ol>
+	 * 1 - Tuning.
+	 * <ol>
+	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
+	 * cfg. Both file should be presented in the running directory.
 	 */
-	public static final boolean tune = false;
+	public static final int tune = 0;
 
 	/**
 	 * Save all configurations tired by open tuner in to

From 0b234b926b86007140d21fcb73ddab339730dd51 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:26:11 +0800
Subject: [PATCH 055/881] Based on the settings, it can evaluate two cfgs.

---
 .../distributed/runtimer/OnlineTuner.java     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4374b466..81688385 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -49,6 +49,16 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
+		if (GlobalConstants.tune == 1)
+			tune();
+		else if (GlobalConstants.tune == 2)
+			evaluate();
+		else
+			System.err
+					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
+	}
+
+	private void tune() {
 		int round = 0;
 		try {
 			tuner.startTuner(String.format(
@@ -104,6 +114,22 @@ public void run() {
 		}
 	}
 
+	private void evaluate() {
+		Configuration finalCfg = readConfiguration(String.format(
+				"final_%s.cfg", app.name));
+		evaluateConfig(finalCfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
+		if (manager.isRunning()) {
+			drainer.startDraining(1);
+		} else {
+			manager.stop();
+		}
+	}
+
 	/**
 	 * @param cfgJson
 	 * @param round

From a834683f4ee18c842d4cebdffcd14449ade03ecd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Feb 2014 14:07:43 +0800
Subject: [PATCH 056/881] ignores first run at final meassurement

First run often shows noise. So we ignore it from the average runtime
calculation.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 81688385..a807a80c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -233,15 +233,18 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
-		int count = 3;
+		int count = 8;
 		try {
 			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
 					true);
 			writer.write("\n----------------------------------------\n");
 			writer.write(String.format("Configuration name = %s\n", cfgName));
 			if (cfg != null) {
+				Pair<Boolean, Long> ret = reconfigure(cfg); // often the first
+															// run shows huge
+															// noise.
 				for (int i = 0; i < count; i++) {
-					Pair<Boolean, Long> ret = reconfigure(cfg);
+					ret = reconfigure(cfg);
 					writer.write(ret.second.toString());
 					writer.write('\n');
 					total += ret.second;

From 81709c45a7cc35c028355fe14246f1c4a299c7dc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 09:35:20 +0800
Subject: [PATCH 057/881] Verifies the configuration which comes from file

Verifies whether the configuration read from file and the default configuration
have same set of parameters. If the parameter set doesn't match, quit
the execution.
---
 .../distributed/DistributedStreamCompiler.java     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index d72c22aa..4e5970fb 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,8 +129,18 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune == 0)
-			this.cfg = readConfiguration(stream.getClass().getSimpleName());
+		if (GlobalConstants.tune == 0) {
+			Configuration cfg1 = readConfiguration(stream.getClass()
+					.getSimpleName());
+			if (!this.cfg.getParametersMap().keySet()
+					.equals(cfg1.getParametersMap().keySet())) {
+				System.err
+						.println("Reading the configuration from configuration file");
+				System.err
+						.println("No matching between parameters in the read configuration and parameters in the default configuration");
+				return null;
+			}
+		}
 
 		if (cfg == null) {
 			System.err

From 85fe85c3403ba5431f0b9ab9fb7af7a9afda8328 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 11:03:06 +0800
Subject: [PATCH 058/881] Integer overflow problem handled

When calculating lcm for buffer sizes, integer overflow occurs. Now all
calculations are done in long.
---
 .../impl/distributed/node/BlobsManagerImpl.java       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 37d72161..e49df5f4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = lcm(minInputBufCapaciy.get(t),
+			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}
@@ -160,7 +160,7 @@ private void addBuffer(Token t, int minSize,
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
-	private int gcd(int a, int b) {
+	private long gcd(long a, long b) {
 		while (true) {
 			if (a == 0)
 				return b;
@@ -171,9 +171,10 @@ private int gcd(int a, int b) {
 		}
 	}
 
-	private int lcm(int a, int b) {
-		int val = gcd(a, b);
-		return val != 0 ? ((a * b) / val) : 0;
+	private long lcm(long a, long b) {
+		long val = gcd(a, b);
+		long quotient = a / val;
+		return val != 0 ? b * quotient : 0;
 	}
 
 	private Set<Token> getLocalTokens(Set<Blob> blobSet) {

From 9c984478469d14b1339363714bba2c82b1f22109 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 14:52:56 +0800
Subject: [PATCH 059/881] lcm -> max

buffer is created with max of input and output buffer sizes not with
lcm of those.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e49df5f4..c177eb71 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
+			int bufSize = Math.max(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}

From c46e3f725e89bc25b07fb04d2014e554750bb12e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 15:01:07 +0800
Subject: [PATCH 060/881] Meaningless variable name has been modified

ss -> connection
sdk -> connection
---
 lib/opentuner/streamjit/onlinetuner.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index d68c14f0..9b14ac66 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,9 +17,9 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, ss, manipulator, inputmanager, objective):
+	def __init__(self, args, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
-		self.sdk = ss
+		self.connection = connection
 		self.trycount = 0
 
 	def run(self, desired_result, input, limit):
@@ -27,11 +27,11 @@ def run(self, desired_result, input, limit):
 		print self.trycount
 		cfg = desired_result.configuration.data
 		#self.niceprint(cfg)
-		self.sdk.sendmsg("%s\n"%cfg)
-		msg = self.sdk.recvmsg()
+		self.connection.sendmsg("%s\n"%cfg)
+		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
-			self.sdk.close()
+			self.connection.close()
 			sys.exit(1)
 		exetime = float(msg)
 		if exetime < 0:
@@ -57,13 +57,13 @@ def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
 		cfg = configuration.data
 		print "Final configuration", cfg
-		self.sdk.sendmsg("Completed")
-		self.sdk.sendmsg("%s\n"%cfg)
-		self.sdk.close()
+		self.connection.sendmsg("Completed")
+		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.close()
 		sys.exit(0)
 
 
-def main(args, cfg, ss):
+def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
@@ -73,13 +73,13 @@ def main(args, cfg, ss):
 		#print "\t", key
   		manipulator.add_parameter(cfg.getParameter(key))
 	
-	mi = StreamJitMI(args, ss, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
 	m.main()
 
-def start(argv, cfg, ss):
+def start(argv, cfg, connection):
 	log = logging.getLogger(__name__)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
 
@@ -90,4 +90,4 @@ def start(argv, cfg, ss):
 	if not args.database:
     		args.database = 'sqlite:///' + args.program + '.db'
 
-	main(args, cfg, ss)
+	main(args, cfg, connection)

From b44b2f0bf66f2913024341a2e0cbd87f4915e6a8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:31:34 +0800
Subject: [PATCH 061/881] Receives Json string from python side of OpenTuner

Previously, python side open tuner was sending a python dictionary and it was
decoded and updated to the configuration object at Java side. Now python side
OT sends Json string.
---
 lib/opentuner/streamjit/onlinetuner.py        | 21 +++++++++++--------
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++--------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 9b14ac66..84e11b8e 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,17 +17,22 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, connection, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.connection = connection
 		self.trycount = 0
+		self.config = configuration
 
 	def run(self, desired_result, input, limit):
 		self.trycount = self.trycount + 1
 		print self.trycount
-		cfg = desired_result.configuration.data
-		#self.niceprint(cfg)
-		self.connection.sendmsg("%s\n"%cfg)
+
+		cfg_data = desired_result.configuration.data
+		#self.niceprint(cfg_data)
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+		self.connection.sendmsg(self.config.toJSON())
+
 		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
@@ -67,13 +72,11 @@ def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = cfg.getAllParameters()
 	#print "\nFeature variables...."
-	for key in params.keys():
-		#print "\t", key
-  		manipulator.add_parameter(cfg.getParameter(key))
+	for p in cfg.getAllParameters().values():
+		manipulator.add_parameter(p)
 	
-	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 55fff3e8..e1fc7910 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -27,7 +27,7 @@
 
 /**
  * Online tuner does continues learning.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Oct 8, 2013
  */
@@ -62,8 +62,8 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			String s = getConfigurationString(app.blobConfiguration);
-			tuner.writeLine(s);
+			// String s = getConfigurationString(app.blobConfiguration);
+			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
@@ -81,8 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				Configuration config = rebuildConfiguration(pythonDict,
-						app.blobConfiguration);
+				// Configuration config = rebuildConfiguration(pythonDict,
+				// app.blobConfiguration);
+
+				Configuration config = Configuration.fromJson(pythonDict);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -153,7 +155,7 @@ public void run() {
 
 	/**
 	 * Just excerpted from run() method for better readability.
-	 *
+	 * 
 	 * @throws IOException
 	 */
 	private void handleTermination() throws IOException {
@@ -174,7 +176,7 @@ private void handleTermination() throws IOException {
 	/**
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
-	 *
+	 * 
 	 * @param pythonDict
 	 */
 	private void runForever(String pythonDict) {
@@ -230,7 +232,7 @@ private void runForever(String pythonDict) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
@@ -279,7 +281,7 @@ private Configuration rebuildConfiguration(String pythonDict,
 	/**
 	 * TODO: This method is totally unnecessary if we remove the usage of the
 	 * name "class" in side {@link Configuration}.
-	 *
+	 * 
 	 * @param cfg
 	 * @return
 	 */

From 5f1ef0ff1078bd302c7072472e81f2c56b77e98e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:37:45 +0800
Subject: [PATCH 062/881] getConfigurationString() has been removed

It is an obselete method.
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e1fc7910..8b076932 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -278,21 +278,6 @@ private Configuration rebuildConfiguration(String pythonDict,
 		return builder.build();
 	}
 
-	/**
-	 * TODO: This method is totally unnecessary if we remove the usage of the
-	 * name "class" in side {@link Configuration}.
-	 * 
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From 9b8c30050b4702ba5d2a59275ebada9984a059e5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:41:29 +0800
Subject: [PATCH 063/881] Renaming: pythonDict -> cfgJson

---
 .../distributed/runtimer/OnlineTuner.java     | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8b076932..ef4a4a74 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -67,13 +67,13 @@ public void run() {
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
-				String pythonDict = tuner.readLine();
-				if (pythonDict == null)
+				String cfgJson = tuner.readLine();
+				if (cfgJson == null)
 					break;
 
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.
-				if (pythonDict.equals("Completed")) {
+				if (cfgJson.equals("Completed")) {
 					handleTermination();
 					break;
 				}
@@ -81,10 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(pythonDict,
+				// Configuration config = rebuildConfiguration(cfgJson,
 				// app.blobConfiguration);
 
-				Configuration config = Configuration.fromJson(pythonDict);
+				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -177,11 +177,11 @@ private void handleTermination() throws IOException {
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 */
-	private void runForever(String pythonDict) {
+	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(pythonDict,
+		Configuration config = rebuildConfiguration(cfgJson,
 				app.blobConfiguration);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
@@ -233,26 +233,26 @@ private void runForever(String pythonDict) {
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
 	 * @param config
 	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
+	 * @return New configuration object with updated values from the cfgJson.
 	 */
-	private Configuration rebuildConfiguration(String pythonDict,
+	private Configuration rebuildConfiguration(String cfgJson,
 			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
+		// System.out.println(cfgJson);
+		checkNotNull(cfgJson, "Received Python dictionary is null");
+		cfgJson = cfgJson.replaceAll("u'", "");
+		cfgJson = cfgJson.replaceAll("':", "");
+		cfgJson = cfgJson.replaceAll("\\{", "");
+		cfgJson = cfgJson.replaceAll("\\}", "");
 		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
 				.trimResults();
 		Configuration.Builder builder = Configuration.builder();
 		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
+		for (String s : dictSplitter.split(cfgJson)) {
 			String[] str = s.split(" ");
 			if (str.length != 2)
 				throw new AssertionError("Wrong python dictionary...");

From 16d95d918d2a23f312bc0634a4c165703b31b7f9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:46:40 +0800
Subject: [PATCH 064/881] rebuildConfiguration() has been removed

This is a obselete method
---
 .../distributed/runtimer/OnlineTuner.java     | 62 +------------------
 1 file changed, 3 insertions(+), 59 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ef4a4a74..8a4550cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -81,8 +81,6 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(cfgJson,
-				// app.blobConfiguration);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
@@ -161,7 +159,8 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		saveConfg(rebuildConfiguration(finalConfg, app.blobConfiguration), 0);
+		Configuration config = Configuration.fromJson(finalConfg);
+		saveConfg(config, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -181,8 +180,7 @@ private void handleTermination() throws IOException {
 	 */
 	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(cfgJson,
-				app.blobConfiguration);
+		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");
@@ -224,60 +222,6 @@ private void runForever(String cfgJson) {
 		}
 	}
 
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param cfgJson
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the cfgJson.
-	 */
-	private Configuration rebuildConfiguration(String cfgJson,
-			Configuration config) {
-		// System.out.println(cfgJson);
-		checkNotNull(cfgJson, "Received Python dictionary is null");
-		cfgJson = cfgJson.replaceAll("u'", "");
-		cfgJson = cfgJson.replaceAll("':", "");
-		cfgJson = cfgJson.replaceAll("\\{", "");
-		cfgJson = cfgJson.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(cfgJson)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-
-		}
-		return builder.build();
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From ccfc3505fe9188742be265976946997bb8f13542 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:49:21 +0800
Subject: [PATCH 065/881] saveConfg() directly takes configuration json...

string as an argument instead of configuration object.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8a4550cc..bc5ec943 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -85,7 +85,7 @@ public void run() {
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(config, tryCount);
+					saveConfg(cfgJson, tryCount);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {
@@ -159,8 +159,7 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		Configuration config = Configuration.fromJson(finalConfg);
-		saveConfg(config, 0);
+		saveConfg(finalConfg, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -225,8 +224,7 @@ private void runForever(String cfgJson) {
 	/**
 	 * Save the configuration.
 	 */
-	private void saveConfg(Configuration config, int round) {
-		String json = config.toJson();
+	private void saveConfg(String json, int round) {
 		try {
 
 			File dir = new File(String.format("configurations%s%s",

From 3a4dbddb791709d0269005cf7f8ebe6287e03e49 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:15:49 +0800
Subject: [PATCH 066/881] Code cleanup

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index bc5ec943..a53e1a9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,21 +1,15 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Splitter;
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -62,7 +56,6 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			// String s = getConfigurationString(app.blobConfiguration);
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");

From 3c0c2c13d6f4d4170f5fa23165fda45d4c622bbf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:37:14 +0800
Subject: [PATCH 067/881] Handles a type of termination condition

termination condition is input data finishes while running with fixed
configuration (i.e., no tuning, GlobalConstants.tune = false).
---
 .../impl/distributed/StreamJitAppManager.java |  1 -
 .../impl/distributed/TailChannel.java         | 34 ++++++++++++++++---
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index a213835e..27e24990 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -245,7 +245,6 @@ else if (GlobalConstants.useDrainData)
 
 		if (isFinal) {
 			this.status = AppStatus.STOPPED;
-			tailChannel.reset();
 			controller.closeAll();
 		}
 		isRunning = false;
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 36f8afc7..fa2e6c3e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Stopwatch;
 
@@ -21,6 +22,8 @@ public class TailChannel extends TCPInputChannel {
 
 	private volatile CountDownLatch latch;
 
+	private performanceLogger pLogger = null;
+
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
 			int limit) {
@@ -28,8 +31,10 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune)
-			new performanceLogger().start();
+		if (!GlobalConstants.tune) {
+			pLogger = new performanceLogger();
+			pLogger.start();
+		}
 	}
 
 	@Override
@@ -41,6 +46,15 @@ public void receiveData() {
 			latch.countDown();
 	}
 
+	@Override
+	public void stop(int type) {
+		super.stop(type);
+		if (pLogger != null) {
+			reset();
+			pLogger.stopLogging();
+		}
+	}
+
 	public void awaitForFixInput() throws InterruptedException {
 		latch.await();
 	}
@@ -53,6 +67,12 @@ public void reset() {
 
 	private class performanceLogger extends Thread {
 
+		private AtomicBoolean stopFlag;
+
+		private performanceLogger() {
+			stopFlag = new AtomicBoolean(false);
+		}
+
 		public void run() {
 			int i = 0;
 			FileWriter writer;
@@ -62,15 +82,15 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30) {
+			while (++i < 30 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();
 					stopwatch.stop();
 					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-					 System.out.println("Execution time is " + time
-					 + " milli seconds");
+					System.out.println("Execution time is " + time
+							+ " milli seconds");
 
 					writer.write(time.toString());
 					writer.write('\n');
@@ -88,5 +108,9 @@ public void run() {
 				e.printStackTrace();
 			}
 		}
+
+		public void stopLogging() {
+			stopFlag.set(true);
+		}
 	}
 }
\ No newline at end of file

From c029e84b7253e766552117b43c790eb3b0e2b9a1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 23:49:31 +0800
Subject: [PATCH 068/881] OutputCount has been added to GlobalConstants.

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 +-
 .../streamjit/impl/distributed/common/GlobalConstants.java  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 27e24990..21348536 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, 1000);
+						+ tailToken.toString(), 0, GlobalConstants.outputCount);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 54f869a4..4215f21b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -99,6 +99,12 @@ private GlobalConstants() {
 	 */
 	public static final boolean saveAllConfigurations = true;
 
+	/**
+	 * Output count for tuning. Tuner measures the running time for this number
+	 * of outputs.
+	 */
+	public static final int outputCount = 100000;
+
 	static {
 
 	}

From d24e834635619413e8be841925d3925deaf832c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 29 Jan 2014 23:24:44 +0800
Subject: [PATCH 069/881] Optional arg -> Positional arg

args.program is necessary
---
 lib/opentuner/streamjit/tuner3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner3.py b/lib/opentuner/streamjit/tuner3.py
index ce8ff3f9..2d5e53d9 100755
--- a/lib/opentuner/streamjit/tuner3.py
+++ b/lib/opentuner/streamjit/tuner3.py
@@ -120,7 +120,7 @@ def make_jvm_options():
 if __name__ == '__main__':
 	logging.basicConfig(level=logging.INFO)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
-	parser.add_argument('--program', help='StreamJIT benchmark to tune (with first input)')
+	parser.add_argument('program', help='StreamJIT benchmark to tune (with first input)')
 	args = parser.parse_args()
 	(cfg_json, error_str) = call_java([], "edu.mit.streamjit.tuner.ConfigGenerator2",
 		["edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory", args.program])

From ccae6dff31bdd4d45f8ac6131b999c60b79784a3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:43:25 +0800
Subject: [PATCH 070/881] Python dict is removed

Uses Json string to communicate with python process back and forth.
---
 lib/opentuner/streamjit/tuner2.py             | 40 +++++++-------
 .../mit/streamjit/tuner/ConfigGenerator.java  |  6 ++-
 src/edu/mit/streamjit/tuner/RunApp.java       | 52 ++++++-------------
 3 files changed, 41 insertions(+), 57 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 4b792432..97ffbf65 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -23,13 +23,14 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, jvmOptions, manipulator, inputmanager, objective):
 		args.technique = ['StreamJITBandit']
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.trycount = 0
 		self.jvmOptions = jvmOptions
 		self.program = args.program
 		self.StreamNodes = []
+		self.config = configuration
 		try:
 			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
 			c = self.tunedataDB.cursor()
@@ -43,15 +44,18 @@ def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
 			data = raw_input ( "Press Keyboard to exit..." )
 
 	def run(self, desired_result, input, limit):
-		cfg = dict.copy(desired_result.configuration.data)
-		(st, t) = self.runApp(cfg)
+		cfg_data = dict.copy(desired_result.configuration.data)
+		(st, t) = self.runApp(cfg_data)
 		return opentuner.resultsdb.models.Result(state=st, time=t)
 
 
-	def runApp(self, cfg):
+	def runApp(self, cfg_data):
 		self.trycount = self.trycount + 1
 		print '\n**********New Run - %d **********'%self.trycount
-		#self.niceprint(cfg)
+		#self.niceprint(cfg_data)
+
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
 
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
@@ -62,7 +66,7 @@ def runApp(self, cfg):
 		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
-			self.jvmOptions.get(key).setValue(cfg[key])
+			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
@@ -72,13 +76,12 @@ def runApp(self, cfg):
 		args.append(str(self.trycount))
 
 		cur = self.tunedataDB.cursor()
-		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, " ".join(jvmArgs), cfg, -1)
+		query = "INSERT INTO results VALUES (%d,'%s','%s', '%f')"%(self.trycount, " ".join(jvmArgs), self.config.toJSON(), -1)
 		cur.execute(query)
 		self.tunedataDB.commit()
-
 		p = subprocess.Popen(args, stderr=subprocess.PIPE)
-		if cfg.get('noOfMachines'):
-			self.startStreamNodes(cfg.get('noOfMachines') - 1, args)
+		if cfg_data.get('noOfMachines'):
+			self.startStreamNodes(cfg_data.get('noOfMachines') - 1, args)
 
 		timeout = 100
 
@@ -103,7 +106,7 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
 			str1 = str(commandStr)
-			str2 = str(cfg)
+			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()
 			return ('ERROR', float('inf'))
@@ -122,11 +125,11 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
-	def niceprint(self, cfg):
+	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount
-		for key in cfg.keys():
-			print "%s - %s"%(key, cfg[key])
+		for key in cfg_data.keys():
+			print "%s - %s"%(key, cfg_data[key])
 
 	def program_name(self):
 		return self.args.program
@@ -166,13 +169,15 @@ def main(args, cfg, jvmOptions):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = dict(cfg.items() + jvmOptions.items())
+	cfgparams = cfg.getAllParameters()
+
+	params = dict(cfgparams.items() + jvmOptions.items())
 	#print "\nFeature variables...."
 	for key in params.keys():
 		#print "\t", key
   		manipulator.add_parameter(params.get(key))
 	
-	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, jvmOptions, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
@@ -200,7 +205,6 @@ def start(program):
 			sys.exit(1)
 		cfgString = row[0]
 		cfg = configuration.getConfiguration(cfgString)
-		cfgparams = cfg.getAllParameters()
 	except Exception, e:
 		print 'Exception occured'
 		traceback.print_exc()
@@ -244,7 +248,7 @@ def start(program):
 	enabledJvmOptions = [aggressiveOpts, compileThreshold, freqInlineSize, maxInlineSize, maxInlineLevel]
 	jvmOptions = {x.name:x for x in enabledJvmOptions}
 
-	main(args, cfgparams, jvmOptions)
+	main(args, cfg, jvmOptions)
 
 if __name__ == '__main__':
 	prgrms = []
diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 3462e2d7..e1abd0de 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,6 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -102,7 +103,8 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		String confString = getConfigurationString(cfg);
+		// String confString = getConfigurationString(cfg);
+		String confString = cfg.toJson();
 
 		try {
 			sqlite.executeUpdate(String.format(
@@ -214,7 +216,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new Compiler2BlobFactory();
+			bf = new CompilerBlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index de342923..42a73049 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
@@ -27,10 +28,10 @@
  * information from streamjit.db based on the passed arguments, runs the
  * streamJit app and update the database with the execution time. StreamJit's
  * opentuner Python script calls this to run the streamJit application.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
- *
+ * 
  */
 public class RunApp {
 
@@ -50,62 +51,39 @@ public static void main(String[] args) throws SQLException {
 		System.out.println(String.format("JAVA Executing: %s Round - %d",
 				benchmarkName, round));
 
-		String dbPath = "streamjit.db";
-
-		sqliteAdapter sqlite;
-		try {
-			sqlite = new sqliteAdapter();
-		} catch (ClassNotFoundException e) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e.printStackTrace();
-			return;
-		}
-		sqlite.connectDB(dbPath);
-
-		ResultSet result = sqlite.executeQuery(String.format(
-				"SELECT * FROM apps WHERE name='%s'", benchmarkName));
-
-		String confgString = result.getString("configuration");
-
 		String sjDbPath = "sj" + benchmarkName + ".db";
 		sqliteAdapter sjDb;
 		try {
 			sjDb = new sqliteAdapter();
 		} catch (ClassNotFoundException e1) {
-			// Actually this exception will not occur. If Sqlite3 did not
-			// exists then it would have exit at previous return point.
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
 			e1.printStackTrace();
 			return;
 		}
-		sjDb.connectDB(sjDbPath);
 
-		ResultSet result1 = sjDb.executeQuery(String.format(
+		sjDb.connectDB(sjDbPath);
+		ResultSet result = sjDb.executeQuery(String.format(
 				"SELECT * FROM results WHERE Round=%d", round));
 
-		String pyDict = result1.getString("SJConfig");
-
-		Configuration config = Configuration
-				.fromJson(getConfigurationString(confgString));
-
-		Configuration cfg2 = rebuildConfiguration(pyDict, config);
+		String cfgJson = result.getString("SJConfig");
+		Configuration cfg = Configuration.fromJson(cfgJson);
 
 		Benchmark app = Benchmarker.getBenchmarkByName(benchmarkName);
 		StreamCompiler sc;
-		IntParameter p = cfg2.getParameter("noOfMachines", IntParameter.class);
+		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
-			csc.configuration(cfg2);
+			CompilerStreamCompiler csc = new CompilerStreamCompiler();
+			csc.setConfig(cfg);
 			sc = csc;
 		} else {
-			sc = new DistributedStreamCompiler(p.getValue(), cfg2);
+			sc = new DistributedStreamCompiler(p.getValue(), cfg);
 		}
 
 		double time = 0;
 		try {
-			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app, sc).get(0);
+			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app,
+					sc).get(0);
 			if (benchmarkResult.isOK())
 				time = benchmarkResult.runMillis();
 			else if (benchmarkResult.kind() == Benchmarker.Result.Kind.TIMEOUT)
@@ -118,7 +96,7 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				time = -2;
 			}
 		} catch (Exception e) {
-			//The Benchmarker should catch everything, but just in case...
+			// The Benchmarker should catch everything, but just in case...
 			e.printStackTrace();
 			time = -2;
 		} catch (OutOfMemoryError er) {
@@ -155,7 +133,7 @@ private static String getConfigurationString(String s) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.

From d22bb3a7364252a9780dc961a9ec7f6e5ef4cc39 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:48:53 +0800
Subject: [PATCH 071/881] Code cleanup

Obselete methods have been removed. These method became obselete after the
last commit.
---
 .../mit/streamjit/tuner/ConfigGenerator.java  | 24 +------
 src/edu/mit/streamjit/tuner/RunApp.java       | 67 -------------------
 2 files changed, 3 insertions(+), 88 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index e1abd0de..414b2a50 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -25,7 +25,6 @@
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
-import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
  * ConfigGenerator generates {@link Configuration} of an application and stores
@@ -33,31 +32,16 @@
  * In this way, Opentuner can start and stop the StreamJit app for each tuning
  * try so that Opentuner can tune JVM parameters such as heapsize,
  * inlinethreshold, GCpausetime, etc as well.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
  */
 
 public class ConfigGenerator {
 
-	/**
-	 * TODO: Need to remove the string "class" from the {@link Configuration}
-	 * jsonifiers. Once it is done, this method can be removed.
-	 *
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Generates configuration for the passed provider.
-	 *
+	 * 
 	 * @param provider
 	 *            Only first benchmark is used to generate configuration. i.e.,
 	 *            only first benchmark will be tuned.
@@ -103,7 +87,6 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		// String confString = getConfigurationString(cfg);
 		String confString = cfg.toJson();
 
 		try {
@@ -141,14 +124,13 @@ public void connectDB(String path) {
 				statement.setQueryTimeout(30); // set timeout to 30 sec.
 
 			} catch (SQLException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
 
 		/**
 		 * Creates table iff it is not exists
-		 *
+		 * 
 		 * @param table
 		 *            Name of the table
 		 * @param signature
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 42a73049..38457a69 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -1,26 +1,18 @@
 package edu.mit.streamjit.tuner;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-import com.google.common.base.Splitter;
-
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
-import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
-import edu.mit.streamjit.test.Datasets;
 import edu.mit.streamjit.tuner.ConfigGenerator.sqliteAdapter;
 
 /**
@@ -117,63 +109,4 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				"UPDATE results SET Exectime=%f WHERE Round=%d", time, round);
 		sjDb.executeUpdate(qry);
 	}
-
-	private static String getConfigurationString(String s) {
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("javaClassPath", "class");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param pythonDict
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
-	 */
-	private static Configuration rebuildConfiguration(String pythonDict,
-			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-		}
-		return builder.build();
-	}
 }

From a63b49e48051d69d919e9e9f1421c3d00376cdab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:59:34 +0800
Subject: [PATCH 072/881] Bug fix

commandStr -> jvmArgs
---
 lib/opentuner/streamjit/tuner2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 97ffbf65..239b086a 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -105,7 +105,7 @@ def runApp(self, cfg_data):
 			print "\033[31;1mException Found\033[0m"
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
-			str1 = str(commandStr)
+			str1 = str(jvmArgs)
 			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()

From 931fa6fa8b0a41902d4c682edae0dcf887a9be32 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:13:50 +0800
Subject: [PATCH 073/881] Saves Json of the final configuration

---
 lib/opentuner/streamjit/tuner2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 239b086a..376c9fa8 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -139,12 +139,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = dict.copy(configuration.data)
+		cfg_data = dict.copy(configuration.data)
+
 		print "\033[32;1mFinal Config...\033[0m"
-		(state, time) = self.runApp(cfg)
+		(state, time) = self.runApp(cfg_data)
 		conn = sqlite3.connect('streamjit.db', 100)
 		cur = conn.cursor()
-		query = 'INSERT INTO FinalResult VALUES ("%s","%s", %d, "%s", "%f")'%(self.program, cfg, self.trycount, state, float(time))
+		query = "INSERT INTO FinalResult VALUES ('%s','%s', %d, '%s', '%f')"%(self.program, self.config.toJSON(), self.trycount, state, float(time))
 		cur.execute(query)
 		conn.commit()
 

From 6931d80baf3440eaa6159e17f03d62f0eea67810 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:39:44 +0800
Subject: [PATCH 074/881] Uses compiler2

---
 src/edu/mit/streamjit/tuner/ConfigGenerator.java | 3 +--
 src/edu/mit/streamjit/tuner/RunApp.java          | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 414b2a50..1d5d1401 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -198,7 +197,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new CompilerBlobFactory();
+			bf = new Compiler2BlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 38457a69..e0ca0db6 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -9,7 +9,7 @@
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
+import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
@@ -65,8 +65,8 @@ public static void main(String[] args) throws SQLException {
 		StreamCompiler sc;
 		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			CompilerStreamCompiler csc = new CompilerStreamCompiler();
-			csc.setConfig(cfg);
+			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
+			csc.configuration(cfg);
 			sc = csc;
 		} else {
 			sc = new DistributedStreamCompiler(p.getValue(), cfg);

From 21b8c21f08ff2ba5d660a2e3bb3715af369d2d84 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 09:58:52 +0800
Subject: [PATCH 075/881] Refactored

Methods getArgs1() and getArgs2() have been added.
---
 lib/opentuner/streamjit/tuner2.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 376c9fa8..63cf021e 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -60,18 +60,15 @@ def runApp(self, cfg_data):
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
 		javaPath = "java"
-		#the classpath, suitable as the value of the '-cp' java argument
-		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.6.2-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
 
-		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
 			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
-		args.extend(jvmArgs)
-		args.append("edu.mit.streamjit.tuner.RunApp")
+
+		args = self.getArgs1(javaPath, jvmArgs)
 		args.append(str(self.program))
 		args.append(str(self.trycount))
 
@@ -125,6 +122,22 @@ def runApp(self, cfg_data):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
+	# Return args that is to run a runnable jar file.
+	def getArgs1(self, javaPath, jvmArgs):
+		args = [javaPath]
+		args.extend(jvmArgs)
+		args.append("-jar")
+		args.append("RunApp.jar")
+		return args
+
+	# Return args that is to run from class file.
+	def getArgs2(self, javaPath, jvmArgs):
+		#the classpath, suitable as the value of the '-cp' java argument
+		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
+		args = [javaPath, "-cp", javaClassPath]
+		args.append("edu.mit.streamjit.tuner.RunApp")
+		return args
+
 	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount

From 60704dc748cedfa1ef69c7b790762e41b75b4125 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 10:03:02 +0800
Subject: [PATCH 076/881] Updated to reflect latest changes

---
 doc/How to tune.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/How to tune.txt b/doc/How to tune.txt
index 34d0d0b6..ff247917 100644
--- a/doc/How to tune.txt	
+++ b/doc/How to tune.txt	
@@ -2,7 +2,7 @@ Go to lib/opentuner/ folder and run venv-bootstrap.py. This will setup a virtual
 
 Run the edu.mit.streamjit.tuner.ConfigGenerator.java to generate configuration information for a particular app. Tuner.java will update the apps table in the streamjit.db with name, configuration, location and class name of the streamjit app.
 
-Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder. If you need to change the input size, you can change it inside the function runApp() in the RunApp class.
+Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder.
 
 Run lib/opentuner/streamjit/tuner2.py.
 

From 6df94a7e496a4215234eca351bd10aaecba6289a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 10:17:36 +0800
Subject: [PATCH 077/881] More organized use of Configuration &  BlobFactory

StreamNodes uses the blobfactory which comes from Configuration.BlobSpecifier.
By fliping GlobalConstants.useCompilerBob flag, we can easily switch between
Interpreter blob and CompilerBlob.
---
 .../distributed/ConfigurationManager.java     | 35 ++++++++++++++++---
 .../distributed/DistributedBlobFactory.java   |  6 ++--
 .../distributed/common/GlobalConstants.java   |  5 +++
 .../node/CfgStringProcessorImpl.java          | 15 ++------
 .../impl/distributed/runtimer/Controller.java | 18 +---------
 5 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 21fe572f..67475cf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -19,9 +20,13 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
@@ -125,17 +130,21 @@ public Configuration getDynamicConfiguration() {
 			PartitionParameter.Builder partParam = PartitionParameter.builder(
 					GlobalConstants.PARTITION, coresPerMachine);
 
-			BlobFactory factory = new Interpreter.InterpreterBlobFactory();
-			partParam.addBlobFactory(factory);
-
+			BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
+			BlobFactory comp2Factory = new Compiler2BlobFactory();
+			partParam.addBlobFactory(intFactory);
+			partParam.addBlobFactory(comp2Factory);
 			app.blobtoMachineMap = new HashMap<>();
 
+			BlobFactory bf = GlobalConstants.useCompilerBlob
+					? comp2Factory
+					: intFactory;
 			for (Integer machineID : app.partitionsMachineMap.keySet()) {
 				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
 						.get(machineID);
 				for (Set<Worker<?, ?>> blobWorkers : blobList) {
 					// TODO: One core per blob. Need to change this.
-					partParam.addBlob(machineID, 1, factory, blobWorkers);
+					partParam.addBlob(machineID, 1, bf, blobWorkers);
 
 					// TODO: Temp fix to build.
 					Token t = Utils.getblobID(blobWorkers);
@@ -144,9 +153,25 @@ public Configuration getDynamicConfiguration() {
 			}
 
 			builder.addParameter(partParam.build());
-			if (app.blobConfiguration != null)
+			if (GlobalConstants.useCompilerBlob)
 				builder.addSubconfiguration("blobConfigs",
 						app.blobConfiguration);
+			else
+				builder.addSubconfiguration("blobConfigs",
+						getInterpreterConfg());
+			return builder.build();
+		}
+
+		private Configuration getInterpreterConfg() {
+			Configuration.Builder builder = Configuration.builder();
+			// TODO: Ensure the need of this switch parameter.
+			List<ChannelFactory> universe = Arrays
+					.<ChannelFactory> asList(new ConcurrentChannelFactory());
+			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
+					"channelFactory", ChannelFactory.class, universe.get(0),
+					universe);
+
+			builder.addParameter(cfParameter);
 			return builder.build();
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 8ded5576..851d15aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -9,6 +9,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
 /**
@@ -61,13 +62,14 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		Configuration distCfg = cfgManager.getDefaultConfiguration(workers,
 				noOfMachines);
-		Configuration.Builder builder = Configuration.builder(distCfg);
+		if (!GlobalConstants.useCompilerBlob)
+			return distCfg;
 
+		Configuration.Builder builder = Configuration.builder(distCfg);
 		BlobFactory compilerBf = new Compiler2BlobFactory();
 		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
-
 		return builder.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 4215f21b..ea7c44ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -105,6 +105,11 @@ private GlobalConstants() {
 	 */
 	public static final int outputCount = 100000;
 
+	/**
+	 * if true uses Compiler2, interpreter otherwise.
+	 */
+	public static final boolean useCompilerBlob = true;
+
 	static {
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index c937d94e..df029b2f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -25,14 +25,12 @@
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter.BlobSpecifier;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
-import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -128,32 +126,25 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			if (blobList == null)
 				return blobSet.build();
 
-			BlobFactory bf;
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
-			if (blobConfigs == null) {
-				blobConfigs = staticConfig;
-				bf = new Interpreter.InterpreterBlobFactory();
-			} else {
-				bf = new Compiler2BlobFactory();
-			}
 
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				// DEBUG
+				// DEBUG MSG
 				System.out.println(String.format(
 						"A new blob with workers %s has been created.",
 						workIdentifiers.toString()));
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 
 				try {
+					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;
 				}
 			}
-
 			return blobSet.build();
 		} else
 			return null;
@@ -200,7 +191,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 		URL url;
 		try {
 			url = jarFile.toURI().toURL();
-			URL[] urls = new URL[] { url };
+			URL[] urls = new URL[]{url};
 
 			ClassLoader loader = new URLClassLoader(urls);
 			Class<?> topStreamClass;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 7d42c667..84533dd1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -13,9 +12,7 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
@@ -29,7 +26,6 @@
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
-import edu.mit.streamjit.impl.interp.ChannelFactory;
 
 /**
  * {@link Controller} controls all {@link StreamNode}s in runtime. It has
@@ -126,25 +122,13 @@ public Map<Integer, Integer> getCoreCount() {
 
 	public void newApp(Configuration staticCfg) {
 		Configuration.Builder builder = Configuration.builder(staticCfg);
-
 		Map<Integer, InetAddress> inetMap = new HashMap<>();
 		for (StreamNodeAgent agent : StreamNodeMap.values())
 			inetMap.put(agent.getNodeID(), agent.getAddress());
 
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
-
-		// TODO: Ensure the need of this switch parameter.
-		List<ChannelFactory> universe = Arrays
-				.<ChannelFactory> asList(new ConcurrentChannelFactory());
-		SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
-				"channelFactory", ChannelFactory.class, universe.get(0),
-				universe);
-
-		builder.addParameter(cfParameter).putExtraData(
-				GlobalConstants.INETADDRESS_MAP, inetMap);
-
+		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
 		this.conProvider = new TCPConnectionProvider(controllerNodeID, inetMap);
-
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
 		sendToAll(json);

From 63840caa28f0d02c1808f6e89b48c0d4297393e7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 13:10:27 +0800
Subject: [PATCH 078/881] Sends json value of final confg

Final config was sent as a python dict. Now changed it to Json format.
I overlooked it.
---
 lib/opentuner/streamjit/onlinetuner.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 84e11b8e..115c971d 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -60,10 +60,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = configuration.data
-		print "Final configuration", cfg
+		cfg_data = configuration.data
+		print "Final configuration", cfg_data
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+
 		self.connection.sendmsg("Completed")
-		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.sendmsg(self.config.toJSON())
 		self.connection.close()
 		sys.exit(0)
 

From 062638d6fa4dc3b10663a8f0db3ea56fc9c8ae48 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 17:01:06 +0800
Subject: [PATCH 079/881] Converts pythonDict to Configuration JSON

---
 .../impl/common/ConfigurationEditor.java      | 136 +++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index f615dcb9..d390ceef 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -1,30 +1,41 @@
 package edu.mit.streamjit.impl.common;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
 
+import com.google.common.base.Splitter;
+
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.BlobFactory;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.fmradio.FMRadio;
+import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
+import edu.mit.streamjit.util.json.Jsonifiers;
 
 public class ConfigurationEditor {
 
 	static String name;
 	static int noofwrks;
+
 	/**
 	 * @param args
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		edit(name, noofwrks);
+		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		 edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
+		// convert();
 	}
 
 	private static void generate(OneToOneElement<?, ?> stream) {
@@ -109,4 +120,127 @@ private static void print(String name) {
 			}
 		}
 	}
+
+	private static void convert() {
+		String appName = "ChannelVocoder7Kernel";
+		Configuration cfg = readConfiguration(String.format("%d%s", 1, appName));
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					String.format("%d%s.cfg", 0, appName)));
+			String pythonDict = reader.readLine();
+
+			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
+			saveConfg(finalCfg, 0);
+		} catch (Exception ex) {
+			ex.printStackTrace();
+		}
+	}
+
+	/**
+	 * Creates a new {@link Configuration} from the received python dictionary
+	 * string. This is not a good way to do.
+	 * <p>
+	 * TODO: Need to add a method to {@link Configuration} so that the
+	 * configuration object can be updated from the python dict string. Now we
+	 * are destructing the old confg object and recreating a new one every time.
+	 * Not a appreciatable way.
+	 * 
+	 * @param pythonDict
+	 *            Python dictionary string. Autotuner gives a dictionary of
+	 *            features with trial values.
+	 * @param config
+	 *            Old configuration object.
+	 * @return New configuration object with updated values from the pythonDict.
+	 */
+	private static Configuration rebuildConfiguration(String pythonDict,
+			Configuration config) {
+		// System.out.println(pythonDict);
+		pythonDict = pythonDict.replaceAll("u'", "");
+		pythonDict = pythonDict.replaceAll("':", "");
+		pythonDict = pythonDict.replaceAll("\\{", "");
+		pythonDict = pythonDict.replaceAll("\\}", "");
+		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
+				.trimResults();
+		Configuration.Builder builder = Configuration.builder();
+		System.out.println("New parameter values from Opentuner...");
+		for (String s : dictSplitter.split(pythonDict)) {
+			String[] str = s.split(" ");
+			if (str.length != 2)
+				throw new AssertionError("Wrong python dictionary...");
+			Parameter p = config.getParameter(str[0]);
+			if (p == null)
+				continue;
+			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
+			if (p instanceof IntParameter) {
+				IntParameter ip = (IntParameter) p;
+				builder.addParameter(new IntParameter(ip.getName(),
+						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
+
+			} else if (p instanceof SwitchParameter<?>) {
+				SwitchParameter sp = (SwitchParameter) p;
+				Class<?> type = sp.getGenericParameter();
+				int val = Integer.parseInt(str[1]);
+				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
+						type, sp.getUniverse().get(val), sp.getUniverse());
+				builder.addParameter(sp1);
+			}
+
+		}
+		return builder.build();
+	}
+
+	/**
+	 * TODO: This method is totally unnecessary if we remove the usage of the
+	 * name "class" in side {@link Configuration}.
+	 * 
+	 * @param cfg
+	 * @return
+	 */
+	private static String getConfigurationString(Configuration cfg) {
+		String s = Jsonifiers.toJson(cfg).toString();
+		String s1 = s.replaceAll("__class__", "ttttt");
+		String s2 = s1.replaceAll("class", "javaClassPath");
+		String s3 = s2.replaceAll("ttttt", "__class__");
+		return s3;
+	}
+
+	/**
+	 * Save the configuration.
+	 */
+	private static void saveConfg(Configuration config, int round) {
+		String json = config.toJson();
+		String name = "erer";
+		try {
+
+			File dir = new File(String.format("configurations%s%s",
+					File.separator, name));
+			if (!dir.exists())
+				if (!dir.mkdirs()) {
+					System.err.println("Make directory failed");
+					return;
+				}
+
+			File file = new File(dir, String.format("%d%s.cfg", round, name));
+			FileWriter writer = new FileWriter(file, false);
+			writer.write(json);
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static Configuration readConfiguration(String simpeName) {
+		String name = String.format("%s.cfg", simpeName);
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
 }

From 717eda45fd9819b690c11991cbd31fd62126036d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 08:48:38 +0800
Subject: [PATCH 080/881] tryCount -> round

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index a53e1a9d..3fac55cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -46,7 +46,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
-		int tryCount = 0;
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -73,12 +73,12 @@ public void run() {
 
 				System.out
 						.println("----------------------------------------------");
-				System.out.println(tryCount++);
+				System.out.println(round++);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, tryCount);
+					saveConfg(cfgJson, round);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {

From 4b7de78b210844cf610622896d3687d94e022514 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 09:20:46 +0800
Subject: [PATCH 081/881] Variable round has been made to global

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 3fac55cc..aefa10f8 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -32,6 +32,7 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
+	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -42,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
+		this.round = 0;
 	}
 
 	@Override
 	public void run() {
-		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,

From 3d968a122092bb4758e5e38fa6a900ab34df16ed Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:36:52 +0800
Subject: [PATCH 082/881] Refactored the run method.

New method reconfigure() has been created.
---
 .../distributed/runtimer/OnlineTuner.java     | 136 ++++++++++--------
 1 file changed, 75 insertions(+), 61 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index aefa10f8..f4c88a2b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
+import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -32,7 +33,6 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
-	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -43,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
-		this.round = 0;
 	}
 
 	@Override
 	public void run() {
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -59,6 +59,8 @@ public void run() {
 			tuner.writeLine("confg");
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
+			Pair<Boolean, Long> ret;
+
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
@@ -72,65 +74,12 @@ public void run() {
 					break;
 				}
 
-				System.out
-						.println("----------------------------------------------");
-				System.out.println(round++);
-
-				Configuration config = Configuration.fromJson(cfgJson);
-
-				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, round);
-
-				try {
-					if (!cfgManager.newConfiguration(config)) {
-						tuner.writeLine("-1");
-						continue;
-					}
-
-					if (manager.isRunning()) {
-						boolean state = drainer.startDraining(0);
-						if (!state) {
-							System.err
-									.println("Final drain has already been called. no more tuning.");
-							tuner.writeLine("exit");
-							break;
-						}
-
-						System.err.println("awaitDrainedIntrmdiate");
-						drainer.awaitDrainedIntrmdiate();
-
-						if (GlobalConstants.useDrainData) {
-							System.err.println("awaitDrainData...");
-							drainer.awaitDrainData();
-							DrainData drainData = drainer.getDrainData();
-							app.drainData = drainData;
-						}
-					}
-
-					drainer.setBlobGraph(app.blobGraph);
-					System.err.println("Reconfiguring...");
-					if (manager.reconfigure()) {
-						Stopwatch stopwatch = Stopwatch.createStarted();
-						manager.awaitForFixInput();
-						stopwatch.stop();
-						// TODO: need to check the manager's status before
-						// passing
-						// the time. Exceptions, final drain, etc may causes app
-						// to
-						// stop executing.
-						long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-
-						System.out.println("Execution time is " + time
-								+ " milli seconds");
-						tuner.writeLine(new Double(time).toString());
-					} else {
-						tuner.writeLine("-1");
-						continue;
-					}
-				} catch (Exception ex) {
-					System.err
-							.println("Couldn't compile the stream graph with this configuration");
-					tuner.writeLine("-1");
+				ret = reconfigure(cfgJson, round++);
+				if (ret.first) {
+					tuner.writeLine(new Double(ret.second).toString());
+				} else {
+					tuner.writeLine("exit");
+					break;
 				}
 			}
 
@@ -145,6 +94,71 @@ public void run() {
 		}
 	}
 
+	/**
+	 * @param cfgJson
+	 * @param round
+	 * @return if ret.first == false, then no more tuning. ret.second = running
+	 *         time in milliseconds.
+	 */
+	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+		long time;
+		System.out.println("----------------------------------------------");
+		System.out.println(round);
+		Configuration config = Configuration.fromJson(cfgJson);
+
+		if (GlobalConstants.saveAllConfigurations || round == 0)
+			saveConfg(cfgJson, round);
+
+		try {
+			if (!cfgManager.newConfiguration(config)) {
+				return new Pair<Boolean, Long>(true, -1l);
+			}
+
+			if (manager.isRunning()) {
+				boolean state = drainer.startDraining(0);
+				if (!state) {
+					System.err
+							.println("Final drain has already been called. no more tuning.");
+					return new Pair<Boolean, Long>(false, -1l);
+				}
+
+				System.err.println("awaitDrainedIntrmdiate");
+				drainer.awaitDrainedIntrmdiate();
+
+				if (GlobalConstants.useDrainData) {
+					System.err.println("awaitDrainData...");
+					drainer.awaitDrainData();
+					DrainData drainData = drainer.getDrainData();
+					app.drainData = drainData;
+				}
+			}
+
+			drainer.setBlobGraph(app.blobGraph);
+			System.err.println("Reconfiguring...");
+			if (manager.reconfigure()) {
+				Stopwatch stopwatch = Stopwatch.createStarted();
+				manager.awaitForFixInput();
+				stopwatch.stop();
+				// TODO: need to check the manager's status before
+				// passing
+				// the time. Exceptions, final drain, etc may causes app
+				// to
+				// stop executing.
+				time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+				System.out.println("Execution time is " + time
+						+ " milli seconds");
+			} else {
+				time = -1l;
+			}
+		} catch (Exception ex) {
+			System.err
+					.println("Couldn't compile the stream graph with this configuration");
+			time = -1l;
+		}
+		return new Pair<Boolean, Long>(true, time);
+	}
+
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 

From e230ef8dd7f6cd8d41d6cbbe796305e508bbde09 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:58:16 +0800
Subject: [PATCH 083/881] reconfiguration() takes only confg as arg

---
 .../distributed/runtimer/OnlineTuner.java     | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f4c88a2b..9a29d767 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,6 +1,8 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
@@ -74,7 +76,15 @@ public void run() {
 					break;
 				}
 
-				ret = reconfigure(cfgJson, round++);
+				System.out
+						.println("----------------------------------------------");
+				System.out.println(round++);
+				Configuration config = Configuration.fromJson(cfgJson);
+
+				if (GlobalConstants.saveAllConfigurations)
+					saveConfg(cfgJson, round);
+
+				ret = reconfigure(config);
 				if (ret.first) {
 					tuner.writeLine(new Double(ret.second).toString());
 				} else {
@@ -100,15 +110,8 @@ public void run() {
 	 * @return if ret.first == false, then no more tuning. ret.second = running
 	 *         time in milliseconds.
 	 */
-	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+	private Pair<Boolean, Long> reconfigure(Configuration config) {
 		long time;
-		System.out.println("----------------------------------------------");
-		System.out.println(round);
-		Configuration config = Configuration.fromJson(cfgJson);
-
-		if (GlobalConstants.saveAllConfigurations || round == 0)
-			saveConfg(cfgJson, round);
-
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				return new Pair<Boolean, Long>(true, -1l);

From d5873353c6e3b9be8100f3416264e6dba3c4c223 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:19:07 +0800
Subject: [PATCH 084/881] Evaluates the final configuration

---
 .../distributed/runtimer/OnlineTuner.java     | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9a29d767..d352c726 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -171,6 +171,14 @@ private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
+
+		Configuration cfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(cfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -232,6 +240,55 @@ private void runForever(String cfgJson) {
 		}
 	}
 
+	/**
+	 * Evaluates a configuration.
+	 * 
+	 * @param cfg
+	 *            configuration that needs to be evaluated
+	 * @param cfgName
+	 *            name of the configuration. This is just for logging purpose.
+	 */
+	private void evaluateConfig(Configuration cfg, String cfgName) {
+		FileWriter writer;
+		double total = 0;
+		int count = 3;
+		try {
+			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
+					true);
+			writer.write("\n----------------------------------------\n");
+			writer.write(String.format("Configuration name = %s\n", cfgName));
+			if (cfg != null) {
+				for (int i = 0; i < count; i++) {
+					Pair<Boolean, Long> ret = reconfigure(cfg);
+					writer.write(ret.second.toString());
+					writer.write('\n');
+					total += ret.second;
+				}
+				double avg = total / count;
+				writer.write(String.format("Average execution time = %f%n\n",
+						avg));
+			} else {
+				writer.write("Null configuration\n");
+			}
+			writer.close();
+		} catch (IOException e1) {
+			e1.printStackTrace();
+		}
+	}
+
+	private Configuration readConfiguration(String name) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
+
 	/**
 	 * Save the configuration.
 	 */

From e8799ee6b790935148bf9730e3bc1331380ef3dd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:22:41 +0800
Subject: [PATCH 085/881] runForever() takes configuration object

previously it took json string of the configuration
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d352c726..f0229b7b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -172,8 +172,8 @@ private void handleTermination() throws IOException {
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
 
-		Configuration cfg = Configuration.fromJson(finalConfg);
-		evaluateConfig(cfg, "Final configuration");
+		Configuration finalcfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
 				app.name));
@@ -186,19 +186,18 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalConfg);
+			runForever(finalcfg);
 		}
 	}
 
 	/**
-	 * TODO: Just copied from the run method. Code duplication between this
-	 * method and the run() method. Try to avoid duplicate code.
+	 * TODO: Just copied from the reconfigure method. Code duplication between
+	 * this method and the reconfigure() method. Try to avoid duplicate code.
 	 * 
-	 * @param cfgJson
+	 * @param config
 	 */
-	private void runForever(String cfgJson) {
+	private void runForever(Configuration config) {
 		System.out.println("runForever");
-		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");

From 631f8d2f7ead463b7091d7cf0bf724888299f8ea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:36:14 +0800
Subject: [PATCH 086/881] Refactored

The method runForever() has been remove and its functionalities are covered
by reconfigure().
---
 .../distributed/runtimer/OnlineTuner.java     | 52 ++-----------------
 1 file changed, 4 insertions(+), 48 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f0229b7b..ed82eb5a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -186,56 +186,12 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalcfg);
-		}
-	}
-
-	/**
-	 * TODO: Just copied from the reconfigure method. Code duplication between
-	 * this method and the reconfigure() method. Try to avoid duplicate code.
-	 * 
-	 * @param config
-	 */
-	private void runForever(Configuration config) {
-		System.out.println("runForever");
-		try {
-			if (!cfgManager.newConfiguration(config)) {
-				System.err.println("Invalid final configuration.");
-				return;
-			}
-
-			if (manager.isRunning()) {
-				boolean state = drainer.startDraining(0);
-				if (!state) {
-					System.err
-							.println("Final drain has already been called. no more tuning.");
-					return;
-				}
-
-				System.err.println("awaitDrainedIntrmdiate");
-				drainer.awaitDrainedIntrmdiate();
-
-				if (GlobalConstants.useDrainData) {
-					System.err.println("awaitDrainData...");
-					drainer.awaitDrainData();
-					DrainData drainData = drainer.getDrainData();
-					app.drainData = drainData;
-				}
-
-				drainer.setBlobGraph(app.blobGraph);
-			}
-
-			System.err.println("Reconfiguring...");
-			boolean var = manager.reconfigure();
-			if (var) {
+			Pair<Boolean, Long> ret = reconfigure(finalcfg);
+			if (ret.first && ret.second > 0)
 				System.out
-						.println("Application is running with the final configuration.");
-			} else {
+						.println("Application is running forever with the final configuration.");
+			else
 				System.err.println("Invalid final configuration.");
-			}
-		} catch (Exception ex) {
-			System.err
-					.println("Couldn't compile the stream graph with this configuration");
 		}
 	}
 

From f293e571c035f7ab51811e961ff3cf8ea3e5496f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 18:53:27 +0800
Subject: [PATCH 087/881] Comments added

---
 .../streamjit/impl/distributed/ConfigurationManager.java   | 7 +++----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java    | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 67475cf3..b4ab69c6 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -34,14 +34,14 @@
  * ConfigurationManager deals with {@link Configuration}. Mainly, It does
  * following two tasks.
  * <ol>
- * <li>Generates configuration for with appropriate tuning parameters for
- * tuning.
+ * <li>Generates configuration with appropriate tuning parameters (Based on the
+ * search space design strategy) for tuning.
  * <li>Dispatch the configuration given by the open tuner and make blobs
  * accordingly.
  * </ol>
  * 
  * One can implement this interface to try different search space designs as
- * they want.
+ * he want.
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Jan 16, 2014
@@ -164,7 +164,6 @@ public Configuration getDynamicConfiguration() {
 
 		private Configuration getInterpreterConfg() {
 			Configuration.Builder builder = Configuration.builder();
-			// TODO: Ensure the need of this switch parameter.
 			List<ChannelFactory> universe = Arrays
 					.<ChannelFactory> asList(new ConcurrentChannelFactory());
 			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index fa2e6c3e..c7e21c94 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -82,7 +82,7 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30 && !stopFlag.get()) {
+			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();

From 82942c061eacf36c6467b8fedae48aab464d11c0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:33:52 +0800
Subject: [PATCH 088/881] Prints status msg

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ed82eb5a..4374b466 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -204,6 +204,7 @@ private void handleTermination() throws IOException {
 	 *            name of the configuration. This is just for logging purpose.
 	 */
 	private void evaluateConfig(Configuration cfg, String cfgName) {
+		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
 		int count = 3;

From 17f6589b521d212a06c34b63d92756d59a97dc19 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:34:40 +0800
Subject: [PATCH 089/881] Print msgs have been commented out

---
 .../impl/distributed/HotSpotTuning.java       | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 4b8a6154..deff1201 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -44,16 +44,16 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	@Override
 	public boolean newConfiguration(Configuration config) {
 
-		for (Parameter p : config.getParametersMap().values()) {
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				System.out.println(ip.getName() + " - " + ip.getValue());
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter<?> sp = (SwitchParameter<?>) p;
-				System.out.println(sp.getName() + " - " + sp.getValue());
-			} else
-				System.out.println(p.getName() + " - Unknown type");
-		}
+		// for (Parameter p : config.getParametersMap().values()) {
+		// if (p instanceof IntParameter) {
+		// IntParameter ip = (IntParameter) p;
+		// System.out.println(ip.getName() + " - " + ip.getValue());
+		// } else if (p instanceof SwitchParameter<?>) {
+		// SwitchParameter<?> sp = (SwitchParameter<?>) p;
+		// System.out.println(sp.getName() + " - " + sp.getValue());
+		// } else
+		// System.out.println(p.getName() + " - Unknown type");
+		// }
 
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
 		try {
@@ -165,11 +165,10 @@ private class PickHotSpots extends StreamVisitor {
 
 		private Joiner<?, ?> skipJoiner;
 
-		private int minSplitjoinSize = 20;
+		private int minSplitjoinSize = 8;
 
 		/**
-		 * Workers those are going to be part {@link OfflineTuner}
-		 * {@link #currentHotSpot}.
+		 * Workers those are going to be part of {@link #currentHotSpot}.
 		 */
 		List<Worker<?, ?>> workerGropups;
 

From 08806e21e22fe3923cd100c9e384ac773a404187 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:13:48 +0800
Subject: [PATCH 090/881] Comments added

---
 .../streamjit/impl/common/AbstractDrainer.java  | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index c7ae3634..35abf078 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -137,15 +137,14 @@ public final void setBlobGraph(BlobGraph blobGraph) {
 	 * Initiate the draining of the blobgraph. Three type of draining could be
 	 * carried out.
 	 * <ol>
-	 * <li>type 0 - Intermediate draining: In this case, no data from input
-	 * buffer will be consumed and StreamJit app will not be stopped. Rather,
-	 * StreamJit app will be just paused for reconfiguration purpose. This
-	 * draining may be triggered by {@link OnlineTuner}.</li>
-	 * <li>type 1 - Semi final draining: In this case, no data from input buffer
-	 * will be consumed but StreamJit app will be stopped. i.e, StreamJit app
-	 * will be stopped safely without consuming any new input. This draining may
-	 * be triggered by {@link OnlineTuner} after opentuner finish tuning and
-	 * send it's final configuration.</li>
+	 * <li>type 0 - Intermediate draining: In this case, no new data from
+	 * {@link Input} will be consumed and StreamJit app will not be stopped.
+	 * Rather, StreamJit app will be just paused for reconfiguration purpose.
+	 * This draining may be triggered by {@link OnlineTuner}.</li>
+	 * <li>type 1 - Semi final draining: In this case, StreamJit app will be
+	 * stopped safely without consuming any new data from {@link Input}. This
+	 * draining may be triggered by {@link OnlineTuner} after opentuner finish
+	 * tuning and send it's final configuration.</li>
 	 * <li>type 2 - Final draining: At the end of input data. After this
 	 * draining StreamJit app will stop. This draining may be triggered by a
 	 * {@link Input} when it run out of input data.</li>

From ae86ec283a699fcf6e4027910a57278ade0d6341 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:15:34 +0800
Subject: [PATCH 091/881] Format changes

---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index d390ceef..2830d72f 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -32,8 +32,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		 edit(name, noofwrks);
+		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}

From 54ec2b24a270f5dccf490bc785e2a02d7ac39439 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:16:28 +0800
Subject: [PATCH 092/881] Deadlock situation at final drain has been handled

Whenever GlobalConstants.useDrainData is false, irrespective of reqDrainData,
stop(3) will be called. This avoids deadlock.
---
 .../impl/distributed/node/BlobsManagerImpl.java      | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2f3ceca8..37d72161 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -298,10 +298,14 @@ private void doDrain(boolean reqDrainData) {
 			drainState = 1;
 
 			for (BoundaryInputChannel bc : inputChannels.values()) {
-				if (!this.reqDrainData)
-					bc.stop(1);
-				else if (GlobalConstants.useDrainData)
-					bc.stop(2);
+				// TODO: [2014-02-05] rearranged this order to call stop(3)
+				// whenever GlobalConstants.useDrainData is false irrespective
+				// of reqDrainData.
+				if (GlobalConstants.useDrainData)
+					if (!this.reqDrainData)
+						bc.stop(1);
+					else
+						bc.stop(2);
 				else
 					bc.stop(3);
 			}

From b0c23eab44d30a558bbdfd990b2a1fefda3bba55 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 22:16:07 +0800
Subject: [PATCH 093/881] Filename changed

"hand_" prefix has been added to hand tuned configuration file generation.
---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 2830d72f..4f6005f6 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -51,7 +51,7 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("%s.cfg", stream.getClass().getSimpleName());
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
 
 		try {
 			FileWriter writer = new FileWriter(name, false);

From 40aaa5963e6d8f0d33a1315c651d12859ea63493 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 6 Feb 2014 00:02:16 +0800
Subject: [PATCH 094/881] New methods to deal with HotSpotTuning searchspace

Generates and edits the configuration files those are created by
HotSpotTuning ConfigurationManager.
---
 .../impl/common/ConfigurationEditor.java      | 90 ++++++++++++++++++-
 1 file changed, 87 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 4f6005f6..52518ccd 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,8 +16,12 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.impl.distributed.HotSpotTuning;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
 import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
 import edu.mit.streamjit.util.json.Jsonifiers;
@@ -32,8 +36,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		edit(name, noofwrks);
+		generate1(new FilterBank6.FilterBankPipeline());
+		edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}
@@ -61,7 +65,6 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
-
 	}
 
 	private static void edit(String name, int maxWor)
@@ -99,6 +102,87 @@ private static void edit(String name, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
+	private static void generate1(OneToOneElement<?, ?> stream) {
+		int noOfnodes = 4;
+
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
+		Worker<?, ?> sink = (Worker<?, ?>) primitiveConnector.getSink();
+		noofwrks = Workers.getIdentifier(sink) + 1;
+
+		StreamJitApp app = new StreamJitApp(stream, source, sink);
+		ConfigurationManager cfgManager = new HotSpotTuning(app);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, noOfnodes);
+
+		Configuration cfg = bf.getDefaultConfiguration(Workers
+				.getAllWorkersInGraph(source));
+
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+
+		try {
+			FileWriter writer = new FileWriter(name, false);
+			writer.write(cfg.toJson());
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static void edit1(String name, int maxWor)
+			throws NumberFormatException, IOException {
+		Configuration cfg;
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			cfg = Configuration.fromJson(json);
+			reader.close();
+		} catch (Exception ex) {
+			System.err.println("File reader error");
+			return;
+		}
+
+		Configuration.Builder builder = Configuration.builder(cfg);
+		BufferedReader keyinreader = new BufferedReader(new InputStreamReader(
+				System.in));
+
+		for (int i = 0; i < maxWor; i++) {
+			String wrkrMachineName = String.format("worker%dtomachine", i);
+			String wrkrCutname = String.format("worker%dcut", i);
+
+			SwitchParameter<Integer> wrkrMachine = cfg.getParameter(
+					wrkrMachineName, SwitchParameter.class);
+			IntParameter wrkrCut = cfg.getParameter(wrkrCutname,
+					IntParameter.class);
+
+			if (wrkrMachine != null) {
+				System.out.println(wrkrMachine.getName() + " - "
+						+ wrkrMachine.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrMachine.getName());
+				builder.addParameter(new SwitchParameter<Integer>(wrkrMachine
+						.getName(), Integer.class, val, wrkrMachine
+						.getUniverse()));
+			}
+
+			if (wrkrCut != null) {
+				System.out.println(wrkrCut.getName() + " - "
+						+ wrkrCut.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrCut.getName());
+				builder.addParameter(new IntParameter(wrkrCut.getName(),
+						wrkrCut.getRange(), val));
+			}
+		}
+
+		cfg = builder.build();
+		FileWriter writer = new FileWriter(name);
+		writer.write(cfg.toJson());
+		writer.close();
+		System.out.println("Successfully updated");
+	}
+
 	private static void print(String name) {
 		Configuration cfg;
 		try {

From e6ce0439ebee5213504929e31e161707eb6d74f8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:12:54 +0800
Subject: [PATCH 095/881] GlobalConstants.tune made to int

Third running mode, evaluates two configuration files mode, has been added.
Previous two are tune and uses cfg file.
---
 .../impl/distributed/DistributedStreamCompiler.java      | 7 ++-----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java  | 2 +-
 .../impl/distributed/common/GlobalConstants.java         | 9 ++++++++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 25f3e799..d72c22aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,11 +129,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune) {
-
-		} else {
+		if (GlobalConstants.tune == 0)
 			this.cfg = readConfiguration(stream.getClass().getSimpleName());
-		}
 
 		if (cfg == null) {
 			System.err
@@ -200,7 +197,7 @@ public void drain() {
 		manager.reconfigure();
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
-		if (GlobalConstants.tune && this.cfg != null) {
+		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
 					cfgManager, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index c7e21c94..13339177 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -31,7 +31,7 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune) {
+		if (GlobalConstants.tune == 0) {
 			pLogger = new performanceLogger();
 			pLogger.start();
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index ea7c44ac..ffd35ad5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -90,8 +90,15 @@ private GlobalConstants() {
 	 * to pass fixed number of input will be measured for 30 rounds and logged
 	 * into FixedOutPut.txt. See {@link TailChannel} for the file logging
 	 * details.
+	 * <ol>
+	 * 0 - No tuning, uses configuration file to run.
+	 * <ol>
+	 * 1 - Tuning.
+	 * <ol>
+	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
+	 * cfg. Both file should be presented in the running directory.
 	 */
-	public static final boolean tune = false;
+	public static final int tune = 0;
 
 	/**
 	 * Save all configurations tired by open tuner in to

From 45aa52ded597eaf45e867d712915b2f2294ef2a6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:26:11 +0800
Subject: [PATCH 096/881] Based on the settings, it can evaluate two cfgs.

---
 .../distributed/runtimer/OnlineTuner.java     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4374b466..81688385 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -49,6 +49,16 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
+		if (GlobalConstants.tune == 1)
+			tune();
+		else if (GlobalConstants.tune == 2)
+			evaluate();
+		else
+			System.err
+					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
+	}
+
+	private void tune() {
 		int round = 0;
 		try {
 			tuner.startTuner(String.format(
@@ -104,6 +114,22 @@ public void run() {
 		}
 	}
 
+	private void evaluate() {
+		Configuration finalCfg = readConfiguration(String.format(
+				"final_%s.cfg", app.name));
+		evaluateConfig(finalCfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
+		if (manager.isRunning()) {
+			drainer.startDraining(1);
+		} else {
+			manager.stop();
+		}
+	}
+
 	/**
 	 * @param cfgJson
 	 * @param round

From 6b358d4b21d05f154eea1258f89135e071dbc6b7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Feb 2014 14:07:43 +0800
Subject: [PATCH 097/881] ignores first run at final meassurement

First run often shows noise. So we ignore it from the average runtime
calculation.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 81688385..a807a80c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -233,15 +233,18 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
-		int count = 3;
+		int count = 8;
 		try {
 			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
 					true);
 			writer.write("\n----------------------------------------\n");
 			writer.write(String.format("Configuration name = %s\n", cfgName));
 			if (cfg != null) {
+				Pair<Boolean, Long> ret = reconfigure(cfg); // often the first
+															// run shows huge
+															// noise.
 				for (int i = 0; i < count; i++) {
-					Pair<Boolean, Long> ret = reconfigure(cfg);
+					ret = reconfigure(cfg);
 					writer.write(ret.second.toString());
 					writer.write('\n');
 					total += ret.second;

From 3e6d96ff21b5fb4df6806228d239b625670a5382 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 09:35:20 +0800
Subject: [PATCH 098/881] Verifies the configuration which comes from file

Verifies whether the configuration read from file and the default configuration
have same set of parameters. If the parameter set doesn't match, quit
the execution.
---
 .../distributed/DistributedStreamCompiler.java     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index d72c22aa..4e5970fb 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,8 +129,18 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune == 0)
-			this.cfg = readConfiguration(stream.getClass().getSimpleName());
+		if (GlobalConstants.tune == 0) {
+			Configuration cfg1 = readConfiguration(stream.getClass()
+					.getSimpleName());
+			if (!this.cfg.getParametersMap().keySet()
+					.equals(cfg1.getParametersMap().keySet())) {
+				System.err
+						.println("Reading the configuration from configuration file");
+				System.err
+						.println("No matching between parameters in the read configuration and parameters in the default configuration");
+				return null;
+			}
+		}
 
 		if (cfg == null) {
 			System.err

From a5c0099ccd8b5d833a60f25fe0e5dde578e77483 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 11:03:06 +0800
Subject: [PATCH 099/881] Integer overflow problem handled

When calculating lcm for buffer sizes, integer overflow occurs. Now all
calculations are done in long.
---
 .../impl/distributed/node/BlobsManagerImpl.java       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 37d72161..e49df5f4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = lcm(minInputBufCapaciy.get(t),
+			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}
@@ -160,7 +160,7 @@ private void addBuffer(Token t, int minSize,
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
-	private int gcd(int a, int b) {
+	private long gcd(long a, long b) {
 		while (true) {
 			if (a == 0)
 				return b;
@@ -171,9 +171,10 @@ private int gcd(int a, int b) {
 		}
 	}
 
-	private int lcm(int a, int b) {
-		int val = gcd(a, b);
-		return val != 0 ? ((a * b) / val) : 0;
+	private long lcm(long a, long b) {
+		long val = gcd(a, b);
+		long quotient = a / val;
+		return val != 0 ? b * quotient : 0;
 	}
 
 	private Set<Token> getLocalTokens(Set<Blob> blobSet) {

From 47125289c9df8208ec46f2b9c2b586a46a3162e0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 14:52:56 +0800
Subject: [PATCH 100/881] lcm -> max

buffer is created with max of input and output buffer sizes not with
lcm of those.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e49df5f4..c177eb71 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
+			int bufSize = Math.max(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}

From 1e14eb4450e4262db35d49a573a8316fc465f105 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Feb 2014 08:11:55 +0800
Subject: [PATCH 101/881] Bug fix

assigning the read configuration to this.cfg.
---
 .../streamjit/impl/distributed/DistributedStreamCompiler.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 4e5970fb..39a8bf48 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -140,6 +140,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 						.println("No matching between parameters in the read configuration and parameters in the default configuration");
 				return null;
 			}
+			this.cfg = cfg1;
 		}
 
 		if (cfg == null) {

From 306956817cbd88bfc655eace17ad93e8d75e7bc3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Feb 2014 10:43:00 +0800
Subject: [PATCH 102/881] Just print msgs added

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 +++
 .../impl/distributed/node/CfgStringProcessorImpl.java          | 1 +
 2 files changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index c177eb71..07d07168 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -157,6 +157,7 @@ private void addBuffer(Token t, int minSize,
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder) {
 		// TODO: Just to increase the performance. Change it later
 		int bufSize = Math.max(1000, minSize);
+		System.out.println("Buffer size of " + t.toString() + " is " + bufSize);
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
@@ -274,6 +275,8 @@ private void start() {
 
 			for (Thread t : blobThreads)
 				t.start();
+
+			System.out.println(blobID + " started");
 		}
 
 		private void stop() {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index df029b2f..692e82dd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -145,6 +145,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					return null;
 				}
 			}
+			System.out.println("All blobs have been created");
 			return blobSet.build();
 		} else
 			return null;

From d4f59dcdccddd0bdbb8d3383ae9ddff8fb1c9a60 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 26 Feb 2014 08:30:20 +0800
Subject: [PATCH 103/881] Renaming: debugPrint -> debugLevel

---
 .../impl/distributed/HeadChannel.java         |  4 ++--
 .../impl/distributed/TailChannel.java         |  4 ++--
 .../distributed/node/TCPInputChannel.java     | 20 +++++++++----------
 .../distributed/node/TCPOutputChannel.java    | 20 +++++++++----------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 0061c2ad..698f029b 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -19,8 +19,8 @@
 public class HeadChannel extends TCPOutputChannel {
 
 	public HeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugPrint);
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 	}
 
 	protected void fillUnprocessedData() {
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 13339177..dc6b197b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -25,9 +25,9 @@ public class TailChannel extends TCPInputChannel {
 	private performanceLogger pLogger = null;
 
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int limit) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugPrint);
+		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index e3b37734..b6c024d2 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -36,7 +36,7 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private final FileWriter writer;
 
-	private final int debugPrint;
+	private final int debugLevel;
 
 	private final Buffer buffer;
 
@@ -61,12 +61,12 @@ public class TCPInputChannel implements BoundaryInputChannel {
 	private ImmutableList<Object> unProcessedData;
 
 	public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
 		this.name = "TCPInputChannel - " + bufferTokenName;
-		this.debugPrint = debugPrint;
+		this.debugLevel = debugLevel;
 		this.softClosed = false;
 		this.extraBuffer = null;
 		this.unProcessedData = null;
@@ -75,7 +75,7 @@ public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		count = 0;
 
 		FileWriter w = null;
-		if (this.debugPrint == 5) {
+		if (this.debugLevel == 5) {
 			try {
 				w = new FileWriter(name, true);
 				w.write("---------------------------------\n");
@@ -147,7 +147,7 @@ public void receiveData() {
 			Object obj = tcpConnection.readObject();
 			count++;
 
-			if (debugPrint == 3) {
+			if (debugLevel == 3) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ obj.toString());
 			}
@@ -158,7 +158,7 @@ public void receiveData() {
 			}
 
 			while (!this.buffer.write(obj)) {
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName()
 							+ " Buffer FULL - " + obj.toString());
 				}
@@ -186,7 +186,7 @@ public void receiveData() {
 				}
 			}
 
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ count + " no of items have been received");
 			}
@@ -230,7 +230,7 @@ private void finalReceive() {
 				Object obj = tcpConnection.readObject();
 				count++;
 
-				if (debugPrint == 2) {
+				if (debugLevel == 2) {
 					System.out.println(Thread.currentThread().getName()
 							+ " finalReceive - " + obj.toString());
 				}
@@ -243,7 +243,7 @@ private void finalReceive() {
 				hasData = true;
 
 				while (!buffer.write(obj)) {
-					if (debugPrint == 3) {
+					if (debugLevel == 3) {
 						System.out.println(Thread.currentThread().getName()
 								+ " finalReceive:Buffer FULL - "
 								+ obj.toString());
@@ -276,7 +276,7 @@ private void finalReceive() {
 					}
 				}
 
-				if (count % 1000 == 0 && debugPrint == 2) {
+				if (count % 1000 == 0 && debugLevel == 2) {
 					System.out.println(Thread.currentThread().getName() + " - "
 							+ count + " no of items have been received");
 				}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index e9e8d8cf..0a74705e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -29,7 +29,7 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	FileWriter writer;
 
-	private final int debugPrint;
+	private final int debugLevel;
 
 	private final Buffer buffer;
 
@@ -50,19 +50,19 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 	protected ImmutableList<Object> unProcessedData;
 
 	public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
 		this.stopFlag = new AtomicBoolean(false);
 		this.isFinal = false;
 		this.name = "TCPOutputChannel - " + bufferTokenName;
-		this.debugPrint = debugPrint;
+		this.debugLevel = debugLevel;
 		this.unProcessedData = null;
 		count = 0;
 
 		FileWriter w = null;
-		if (this.debugPrint == 5) {
+		if (this.debugLevel == 5) {
 			try {
 				w = new FileWriter(name, true);
 				w.write("---------------------------------\n");
@@ -120,7 +120,7 @@ public void run() {
 					}
 				}
 
-				if (debugPrint > 0) {
+				if (debugLevel > 0) {
 					System.err.println(Thread.currentThread().getName()
 							+ " - Exiting...");
 					System.out.println("isFinal " + isFinal);
@@ -137,7 +137,7 @@ public final void sendData() {
 				tcpConnection.writeObject(obj);
 				count++;
 
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName() + " - "
 							+ obj.toString());
 				}
@@ -151,7 +151,7 @@ public final void sendData() {
 						.println("TCP Output Channel. WriteObject exception.");
 				reConnect();
 			}
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ count + " items have been sent");
 			}
@@ -165,7 +165,7 @@ public final int getOtherNodeID() {
 
 	@Override
 	public final void stop(boolean isFinal) {
-		if (debugPrint > 0)
+		if (debugLevel > 0)
 			System.out.println(Thread.currentThread().getName()
 					+ " - stop request");
 		this.isFinal = isFinal;
@@ -183,7 +183,7 @@ private void finalSend() {
 				tcpConnection.writeObject(o);
 				count++;
 
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName()
 							+ " FinalSend - " + o.toString());
 				}
@@ -196,7 +196,7 @@ private void finalSend() {
 			} catch (IOException e) {
 				System.err.println("TCP Output Channel. finalSend exception.");
 			}
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName()
 						+ " FinalSend - " + count
 						+ " no of items have been sent");

From 73b149b6b363e53c0885f4cd5ee0966860ed69c4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 15:01:07 +0800
Subject: [PATCH 104/881] Meaningless variable name has been modified

ss -> connection
sdk -> connection
---
 lib/opentuner/streamjit/onlinetuner.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index d68c14f0..9b14ac66 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,9 +17,9 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, ss, manipulator, inputmanager, objective):
+	def __init__(self, args, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
-		self.sdk = ss
+		self.connection = connection
 		self.trycount = 0
 
 	def run(self, desired_result, input, limit):
@@ -27,11 +27,11 @@ def run(self, desired_result, input, limit):
 		print self.trycount
 		cfg = desired_result.configuration.data
 		#self.niceprint(cfg)
-		self.sdk.sendmsg("%s\n"%cfg)
-		msg = self.sdk.recvmsg()
+		self.connection.sendmsg("%s\n"%cfg)
+		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
-			self.sdk.close()
+			self.connection.close()
 			sys.exit(1)
 		exetime = float(msg)
 		if exetime < 0:
@@ -57,13 +57,13 @@ def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
 		cfg = configuration.data
 		print "Final configuration", cfg
-		self.sdk.sendmsg("Completed")
-		self.sdk.sendmsg("%s\n"%cfg)
-		self.sdk.close()
+		self.connection.sendmsg("Completed")
+		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.close()
 		sys.exit(0)
 
 
-def main(args, cfg, ss):
+def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
@@ -73,13 +73,13 @@ def main(args, cfg, ss):
 		#print "\t", key
   		manipulator.add_parameter(cfg.getParameter(key))
 	
-	mi = StreamJitMI(args, ss, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
 	m.main()
 
-def start(argv, cfg, ss):
+def start(argv, cfg, connection):
 	log = logging.getLogger(__name__)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
 
@@ -90,4 +90,4 @@ def start(argv, cfg, ss):
 	if not args.database:
     		args.database = 'sqlite:///' + args.program + '.db'
 
-	main(args, cfg, ss)
+	main(args, cfg, connection)

From 6aa5c23b4c5852c37d440e5c8ab542fb142b105e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:31:34 +0800
Subject: [PATCH 105/881] Receives Json string from python side of OpenTuner

Previously, python side open tuner was sending a python dictionary and it was
decoded and updated to the configuration object at Java side. Now python side
OT sends Json string.
---
 lib/opentuner/streamjit/onlinetuner.py        | 21 +++++++++++--------
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++--------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 9b14ac66..84e11b8e 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -17,17 +17,22 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, connection, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, connection, manipulator, inputmanager, objective):
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.connection = connection
 		self.trycount = 0
+		self.config = configuration
 
 	def run(self, desired_result, input, limit):
 		self.trycount = self.trycount + 1
 		print self.trycount
-		cfg = desired_result.configuration.data
-		#self.niceprint(cfg)
-		self.connection.sendmsg("%s\n"%cfg)
+
+		cfg_data = desired_result.configuration.data
+		#self.niceprint(cfg_data)
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+		self.connection.sendmsg(self.config.toJSON())
+
 		msg = self.connection.recvmsg()
 		if (msg == "exit\n"):
 			#data = raw_input ( "exit cmd received. Press Keyboard to exit..." )
@@ -67,13 +72,11 @@ def main(args, cfg, connection):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = cfg.getAllParameters()
 	#print "\nFeature variables...."
-	for key in params.keys():
-		#print "\t", key
-  		manipulator.add_parameter(cfg.getParameter(key))
+	for p in cfg.getAllParameters().values():
+		manipulator.add_parameter(p)
 	
-	mi = StreamJitMI(args, connection, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, connection, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 55fff3e8..e1fc7910 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -27,7 +27,7 @@
 
 /**
  * Online tuner does continues learning.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Oct 8, 2013
  */
@@ -62,8 +62,8 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			String s = getConfigurationString(app.blobConfiguration);
-			tuner.writeLine(s);
+			// String s = getConfigurationString(app.blobConfiguration);
+			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
@@ -81,8 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				Configuration config = rebuildConfiguration(pythonDict,
-						app.blobConfiguration);
+				// Configuration config = rebuildConfiguration(pythonDict,
+				// app.blobConfiguration);
+
+				Configuration config = Configuration.fromJson(pythonDict);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -153,7 +155,7 @@ public void run() {
 
 	/**
 	 * Just excerpted from run() method for better readability.
-	 *
+	 * 
 	 * @throws IOException
 	 */
 	private void handleTermination() throws IOException {
@@ -174,7 +176,7 @@ private void handleTermination() throws IOException {
 	/**
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
-	 *
+	 * 
 	 * @param pythonDict
 	 */
 	private void runForever(String pythonDict) {
@@ -230,7 +232,7 @@ private void runForever(String pythonDict) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
@@ -279,7 +281,7 @@ private Configuration rebuildConfiguration(String pythonDict,
 	/**
 	 * TODO: This method is totally unnecessary if we remove the usage of the
 	 * name "class" in side {@link Configuration}.
-	 *
+	 * 
 	 * @param cfg
 	 * @return
 	 */

From a41b4180124c9bc6d881ea32a593532c13e881a8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:37:45 +0800
Subject: [PATCH 106/881] getConfigurationString() has been removed

It is an obselete method.
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e1fc7910..8b076932 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -278,21 +278,6 @@ private Configuration rebuildConfiguration(String pythonDict,
 		return builder.build();
 	}
 
-	/**
-	 * TODO: This method is totally unnecessary if we remove the usage of the
-	 * name "class" in side {@link Configuration}.
-	 * 
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From f5021db5b3b711a00c3b6c34d482a0f03c02cd68 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:41:29 +0800
Subject: [PATCH 107/881] Renaming: pythonDict -> cfgJson

---
 .../distributed/runtimer/OnlineTuner.java     | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8b076932..ef4a4a74 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -67,13 +67,13 @@ public void run() {
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
-				String pythonDict = tuner.readLine();
-				if (pythonDict == null)
+				String cfgJson = tuner.readLine();
+				if (cfgJson == null)
 					break;
 
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.
-				if (pythonDict.equals("Completed")) {
+				if (cfgJson.equals("Completed")) {
 					handleTermination();
 					break;
 				}
@@ -81,10 +81,10 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(pythonDict,
+				// Configuration config = rebuildConfiguration(cfgJson,
 				// app.blobConfiguration);
 
-				Configuration config = Configuration.fromJson(pythonDict);
+				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
 					saveConfg(config, tryCount);
@@ -177,11 +177,11 @@ private void handleTermination() throws IOException {
 	 * TODO: Just copied from the run method. Code duplication between this
 	 * method and the run() method. Try to avoid duplicate code.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 */
-	private void runForever(String pythonDict) {
+	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(pythonDict,
+		Configuration config = rebuildConfiguration(cfgJson,
 				app.blobConfiguration);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
@@ -233,26 +233,26 @@ private void runForever(String pythonDict) {
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
 	 * 
-	 * @param pythonDict
+	 * @param cfgJson
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.
 	 * @param config
 	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
+	 * @return New configuration object with updated values from the cfgJson.
 	 */
-	private Configuration rebuildConfiguration(String pythonDict,
+	private Configuration rebuildConfiguration(String cfgJson,
 			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
+		// System.out.println(cfgJson);
+		checkNotNull(cfgJson, "Received Python dictionary is null");
+		cfgJson = cfgJson.replaceAll("u'", "");
+		cfgJson = cfgJson.replaceAll("':", "");
+		cfgJson = cfgJson.replaceAll("\\{", "");
+		cfgJson = cfgJson.replaceAll("\\}", "");
 		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
 				.trimResults();
 		Configuration.Builder builder = Configuration.builder();
 		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
+		for (String s : dictSplitter.split(cfgJson)) {
 			String[] str = s.split(" ");
 			if (str.length != 2)
 				throw new AssertionError("Wrong python dictionary...");

From 7099111da070b54711f2613ea121d3fc3ec46648 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:46:40 +0800
Subject: [PATCH 108/881] rebuildConfiguration() has been removed

This is a obselete method
---
 .../distributed/runtimer/OnlineTuner.java     | 62 +------------------
 1 file changed, 3 insertions(+), 59 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ef4a4a74..8a4550cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -81,8 +81,6 @@ public void run() {
 				System.out
 						.println("----------------------------------------------");
 				System.out.println(tryCount++);
-				// Configuration config = rebuildConfiguration(cfgJson,
-				// app.blobConfiguration);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
@@ -161,7 +159,8 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		saveConfg(rebuildConfiguration(finalConfg, app.blobConfiguration), 0);
+		Configuration config = Configuration.fromJson(finalConfg);
+		saveConfg(config, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -181,8 +180,7 @@ private void handleTermination() throws IOException {
 	 */
 	private void runForever(String cfgJson) {
 		System.out.println("runForever");
-		Configuration config = rebuildConfiguration(cfgJson,
-				app.blobConfiguration);
+		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");
@@ -224,60 +222,6 @@ private void runForever(String cfgJson) {
 		}
 	}
 
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param cfgJson
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the cfgJson.
-	 */
-	private Configuration rebuildConfiguration(String cfgJson,
-			Configuration config) {
-		// System.out.println(cfgJson);
-		checkNotNull(cfgJson, "Received Python dictionary is null");
-		cfgJson = cfgJson.replaceAll("u'", "");
-		cfgJson = cfgJson.replaceAll("':", "");
-		cfgJson = cfgJson.replaceAll("\\{", "");
-		cfgJson = cfgJson.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(cfgJson)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-
-		}
-		return builder.build();
-	}
-
 	/**
 	 * Save the configuration.
 	 */

From 1c7745de8cf7135952bc7e6d15178da1ae6f9c6f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 16:49:21 +0800
Subject: [PATCH 109/881] saveConfg() directly takes configuration json...

string as an argument instead of configuration object.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8a4550cc..bc5ec943 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -85,7 +85,7 @@ public void run() {
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(config, tryCount);
+					saveConfg(cfgJson, tryCount);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {
@@ -159,8 +159,7 @@ public void run() {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		Configuration config = Configuration.fromJson(finalConfg);
-		saveConfg(config, 0);
+		saveConfg(finalConfg, 0);
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -225,8 +224,7 @@ private void runForever(String cfgJson) {
 	/**
 	 * Save the configuration.
 	 */
-	private void saveConfg(Configuration config, int round) {
-		String json = config.toJson();
+	private void saveConfg(String json, int round) {
 		try {
 
 			File dir = new File(String.format("configurations%s%s",

From 7fa8c28ec846c876d95758b91aa315924eff5168 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:15:49 +0800
Subject: [PATCH 110/881] Code cleanup

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index bc5ec943..a53e1a9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,21 +1,15 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Splitter;
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -62,7 +56,6 @@ public void run() {
 			tuner.writeLine(app.name);
 
 			tuner.writeLine("confg");
-			// String s = getConfigurationString(app.blobConfiguration);
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
 			System.out.println("New tune run.............");

From 50a7451238c5851be2155b91235d5f6836309d43 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 18:37:14 +0800
Subject: [PATCH 111/881] Handles a type of termination condition

termination condition is input data finishes while running with fixed
configuration (i.e., no tuning, GlobalConstants.tune = false).
---
 .../impl/distributed/StreamJitAppManager.java |  1 -
 .../impl/distributed/TailChannel.java         | 34 ++++++++++++++++---
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index a213835e..27e24990 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -245,7 +245,6 @@ else if (GlobalConstants.useDrainData)
 
 		if (isFinal) {
 			this.status = AppStatus.STOPPED;
-			tailChannel.reset();
 			controller.closeAll();
 		}
 		isRunning = false;
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 36f8afc7..fa2e6c3e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Stopwatch;
 
@@ -21,6 +22,8 @@ public class TailChannel extends TCPInputChannel {
 
 	private volatile CountDownLatch latch;
 
+	private performanceLogger pLogger = null;
+
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
 			int limit) {
@@ -28,8 +31,10 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune)
-			new performanceLogger().start();
+		if (!GlobalConstants.tune) {
+			pLogger = new performanceLogger();
+			pLogger.start();
+		}
 	}
 
 	@Override
@@ -41,6 +46,15 @@ public void receiveData() {
 			latch.countDown();
 	}
 
+	@Override
+	public void stop(int type) {
+		super.stop(type);
+		if (pLogger != null) {
+			reset();
+			pLogger.stopLogging();
+		}
+	}
+
 	public void awaitForFixInput() throws InterruptedException {
 		latch.await();
 	}
@@ -53,6 +67,12 @@ public void reset() {
 
 	private class performanceLogger extends Thread {
 
+		private AtomicBoolean stopFlag;
+
+		private performanceLogger() {
+			stopFlag = new AtomicBoolean(false);
+		}
+
 		public void run() {
 			int i = 0;
 			FileWriter writer;
@@ -62,15 +82,15 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30) {
+			while (++i < 30 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();
 					stopwatch.stop();
 					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
-					 System.out.println("Execution time is " + time
-					 + " milli seconds");
+					System.out.println("Execution time is " + time
+							+ " milli seconds");
 
 					writer.write(time.toString());
 					writer.write('\n');
@@ -88,5 +108,9 @@ public void run() {
 				e.printStackTrace();
 			}
 		}
+
+		public void stopLogging() {
+			stopFlag.set(true);
+		}
 	}
 }
\ No newline at end of file

From 544f5f2691803bb35111bd9d480494df2a92a08c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 28 Jan 2014 23:49:31 +0800
Subject: [PATCH 112/881] OutputCount has been added to GlobalConstants.

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 +-
 .../streamjit/impl/distributed/common/GlobalConstants.java  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 27e24990..21348536 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, 1000);
+						+ tailToken.toString(), 0, GlobalConstants.outputCount);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 54f869a4..4215f21b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -99,6 +99,12 @@ private GlobalConstants() {
 	 */
 	public static final boolean saveAllConfigurations = true;
 
+	/**
+	 * Output count for tuning. Tuner measures the running time for this number
+	 * of outputs.
+	 */
+	public static final int outputCount = 100000;
+
 	static {
 
 	}

From 374a342b643188014cbaed100e95956c1aee6c08 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 29 Jan 2014 23:24:44 +0800
Subject: [PATCH 113/881] Optional arg -> Positional arg

args.program is necessary
---
 lib/opentuner/streamjit/tuner3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner3.py b/lib/opentuner/streamjit/tuner3.py
index f4d8f1d5..ebe93ee7 100755
--- a/lib/opentuner/streamjit/tuner3.py
+++ b/lib/opentuner/streamjit/tuner3.py
@@ -129,7 +129,7 @@ def make_jvm_options():
 if __name__ == '__main__':
 	logging.basicConfig(level=logging.INFO)
 	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
-	parser.add_argument('--program', help='StreamJIT benchmark to tune (with first input)')
+	parser.add_argument('program', help='StreamJIT benchmark to tune (with first input)')
 	args = parser.parse_args()
 	(cfg_json, error_str) = call_java([], "edu.mit.streamjit.tuner.ConfigGenerator2",
 		["edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory", args.program])

From 36eed41d1b281812ffe7c7c4348df7dddf8b6f9b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:43:25 +0800
Subject: [PATCH 114/881] Python dict is removed

Uses Json string to communicate with python process back and forth.
---
 lib/opentuner/streamjit/tuner2.py             | 40 +++++++-------
 .../mit/streamjit/tuner/ConfigGenerator.java  |  6 ++-
 src/edu/mit/streamjit/tuner/RunApp.java       | 52 ++++++-------------
 3 files changed, 41 insertions(+), 57 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 4b792432..97ffbf65 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -23,13 +23,14 @@
 
 class StreamJitMI(MeasurementInterface):
 	''' Measurement Interface for tunning a StreamJit application'''
-	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
+	def __init__(self, args, configuration, jvmOptions, manipulator, inputmanager, objective):
 		args.technique = ['StreamJITBandit']
 		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
 		self.trycount = 0
 		self.jvmOptions = jvmOptions
 		self.program = args.program
 		self.StreamNodes = []
+		self.config = configuration
 		try:
 			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
 			c = self.tunedataDB.cursor()
@@ -43,15 +44,18 @@ def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
 			data = raw_input ( "Press Keyboard to exit..." )
 
 	def run(self, desired_result, input, limit):
-		cfg = dict.copy(desired_result.configuration.data)
-		(st, t) = self.runApp(cfg)
+		cfg_data = dict.copy(desired_result.configuration.data)
+		(st, t) = self.runApp(cfg_data)
 		return opentuner.resultsdb.models.Result(state=st, time=t)
 
 
-	def runApp(self, cfg):
+	def runApp(self, cfg_data):
 		self.trycount = self.trycount + 1
 		print '\n**********New Run - %d **********'%self.trycount
-		#self.niceprint(cfg)
+		#self.niceprint(cfg_data)
+
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
 
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
@@ -62,7 +66,7 @@ def runApp(self, cfg):
 		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
-			self.jvmOptions.get(key).setValue(cfg[key])
+			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
@@ -72,13 +76,12 @@ def runApp(self, cfg):
 		args.append(str(self.trycount))
 
 		cur = self.tunedataDB.cursor()
-		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, " ".join(jvmArgs), cfg, -1)
+		query = "INSERT INTO results VALUES (%d,'%s','%s', '%f')"%(self.trycount, " ".join(jvmArgs), self.config.toJSON(), -1)
 		cur.execute(query)
 		self.tunedataDB.commit()
-
 		p = subprocess.Popen(args, stderr=subprocess.PIPE)
-		if cfg.get('noOfMachines'):
-			self.startStreamNodes(cfg.get('noOfMachines') - 1, args)
+		if cfg_data.get('noOfMachines'):
+			self.startStreamNodes(cfg_data.get('noOfMachines') - 1, args)
 
 		timeout = 100
 
@@ -103,7 +106,7 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
 			str1 = str(commandStr)
-			str2 = str(cfg)
+			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()
 			return ('ERROR', float('inf'))
@@ -122,11 +125,11 @@ def runApp(self, cfg):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
-	def niceprint(self, cfg):
+	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount
-		for key in cfg.keys():
-			print "%s - %s"%(key, cfg[key])
+		for key in cfg_data.keys():
+			print "%s - %s"%(key, cfg_data[key])
 
 	def program_name(self):
 		return self.args.program
@@ -166,13 +169,15 @@ def main(args, cfg, jvmOptions):
 	logging.basicConfig(level=logging.INFO)
 	manipulator = ConfigurationManipulator()
 
-	params = dict(cfg.items() + jvmOptions.items())
+	cfgparams = cfg.getAllParameters()
+
+	params = dict(cfgparams.items() + jvmOptions.items())
 	#print "\nFeature variables...."
 	for key in params.keys():
 		#print "\t", key
   		manipulator.add_parameter(params.get(key))
 	
-	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
+	mi = StreamJitMI(args, cfg, jvmOptions, manipulator, FixedInputManager(),
                     MinimizeTime())
 
 	m = TuningRunMain(mi, args)
@@ -200,7 +205,6 @@ def start(program):
 			sys.exit(1)
 		cfgString = row[0]
 		cfg = configuration.getConfiguration(cfgString)
-		cfgparams = cfg.getAllParameters()
 	except Exception, e:
 		print 'Exception occured'
 		traceback.print_exc()
@@ -244,7 +248,7 @@ def start(program):
 	enabledJvmOptions = [aggressiveOpts, compileThreshold, freqInlineSize, maxInlineSize, maxInlineLevel]
 	jvmOptions = {x.name:x for x in enabledJvmOptions}
 
-	main(args, cfgparams, jvmOptions)
+	main(args, cfg, jvmOptions)
 
 if __name__ == '__main__':
 	prgrms = []
diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 3462e2d7..e1abd0de 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,6 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -102,7 +103,8 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		String confString = getConfigurationString(cfg);
+		// String confString = getConfigurationString(cfg);
+		String confString = cfg.toJson();
 
 		try {
 			sqlite.executeUpdate(String.format(
@@ -214,7 +216,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new Compiler2BlobFactory();
+			bf = new CompilerBlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index de342923..42a73049 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
@@ -27,10 +28,10 @@
  * information from streamjit.db based on the passed arguments, runs the
  * streamJit app and update the database with the execution time. StreamJit's
  * opentuner Python script calls this to run the streamJit application.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
- *
+ * 
  */
 public class RunApp {
 
@@ -50,62 +51,39 @@ public static void main(String[] args) throws SQLException {
 		System.out.println(String.format("JAVA Executing: %s Round - %d",
 				benchmarkName, round));
 
-		String dbPath = "streamjit.db";
-
-		sqliteAdapter sqlite;
-		try {
-			sqlite = new sqliteAdapter();
-		} catch (ClassNotFoundException e) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e.printStackTrace();
-			return;
-		}
-		sqlite.connectDB(dbPath);
-
-		ResultSet result = sqlite.executeQuery(String.format(
-				"SELECT * FROM apps WHERE name='%s'", benchmarkName));
-
-		String confgString = result.getString("configuration");
-
 		String sjDbPath = "sj" + benchmarkName + ".db";
 		sqliteAdapter sjDb;
 		try {
 			sjDb = new sqliteAdapter();
 		} catch (ClassNotFoundException e1) {
-			// Actually this exception will not occur. If Sqlite3 did not
-			// exists then it would have exit at previous return point.
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
 			e1.printStackTrace();
 			return;
 		}
-		sjDb.connectDB(sjDbPath);
 
-		ResultSet result1 = sjDb.executeQuery(String.format(
+		sjDb.connectDB(sjDbPath);
+		ResultSet result = sjDb.executeQuery(String.format(
 				"SELECT * FROM results WHERE Round=%d", round));
 
-		String pyDict = result1.getString("SJConfig");
-
-		Configuration config = Configuration
-				.fromJson(getConfigurationString(confgString));
-
-		Configuration cfg2 = rebuildConfiguration(pyDict, config);
+		String cfgJson = result.getString("SJConfig");
+		Configuration cfg = Configuration.fromJson(cfgJson);
 
 		Benchmark app = Benchmarker.getBenchmarkByName(benchmarkName);
 		StreamCompiler sc;
-		IntParameter p = cfg2.getParameter("noOfMachines", IntParameter.class);
+		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
-			csc.configuration(cfg2);
+			CompilerStreamCompiler csc = new CompilerStreamCompiler();
+			csc.setConfig(cfg);
 			sc = csc;
 		} else {
-			sc = new DistributedStreamCompiler(p.getValue(), cfg2);
+			sc = new DistributedStreamCompiler(p.getValue(), cfg);
 		}
 
 		double time = 0;
 		try {
-			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app, sc).get(0);
+			Benchmarker.Result benchmarkResult = Benchmarker.runBenchmark(app,
+					sc).get(0);
 			if (benchmarkResult.isOK())
 				time = benchmarkResult.runMillis();
 			else if (benchmarkResult.kind() == Benchmarker.Result.Kind.TIMEOUT)
@@ -118,7 +96,7 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				time = -2;
 			}
 		} catch (Exception e) {
-			//The Benchmarker should catch everything, but just in case...
+			// The Benchmarker should catch everything, but just in case...
 			e.printStackTrace();
 			time = -2;
 		} catch (OutOfMemoryError er) {
@@ -155,7 +133,7 @@ private static String getConfigurationString(String s) {
 	 * configuration object can be updated from the python dict string. Now we
 	 * are destructing the old confg object and recreating a new one every time.
 	 * Not a appreciatable way.
-	 *
+	 * 
 	 * @param pythonDict
 	 *            Python dictionary string. Autotuner gives a dictionary of
 	 *            features with trial values.

From 2bb74c44cab99d0b7db4294a8edbeb7c54e1ff90 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:48:53 +0800
Subject: [PATCH 115/881] Code cleanup

Obselete methods have been removed. These method became obselete after the
last commit.
---
 .../mit/streamjit/tuner/ConfigGenerator.java  | 24 +------
 src/edu/mit/streamjit/tuner/RunApp.java       | 67 -------------------
 2 files changed, 3 insertions(+), 88 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index e1abd0de..414b2a50 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -25,7 +25,6 @@
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
-import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
  * ConfigGenerator generates {@link Configuration} of an application and stores
@@ -33,31 +32,16 @@
  * In this way, Opentuner can start and stop the StreamJit app for each tuning
  * try so that Opentuner can tune JVM parameters such as heapsize,
  * inlinethreshold, GCpausetime, etc as well.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 10, 2013
  */
 
 public class ConfigGenerator {
 
-	/**
-	 * TODO: Need to remove the string "class" from the {@link Configuration}
-	 * jsonifiers. Once it is done, this method can be removed.
-	 *
-	 * @param cfg
-	 * @return
-	 */
-	private String getConfigurationString(Configuration cfg) {
-		String s = Jsonifiers.toJson(cfg).toString();
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("class", "javaClassPath");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
 	/**
 	 * Generates configuration for the passed provider.
-	 *
+	 * 
 	 * @param provider
 	 *            Only first benchmark is used to generate configuration. i.e.,
 	 *            only first benchmark will be tuned.
@@ -103,7 +87,6 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		Configuration cfg = factory.getDefaultConfiguration(workers);
 
 		String name = app.toString();
-		// String confString = getConfigurationString(cfg);
 		String confString = cfg.toJson();
 
 		try {
@@ -141,14 +124,13 @@ public void connectDB(String path) {
 				statement.setQueryTimeout(30); // set timeout to 30 sec.
 
 			} catch (SQLException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
 
 		/**
 		 * Creates table iff it is not exists
-		 *
+		 * 
 		 * @param table
 		 *            Name of the table
 		 * @param signature
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 42a73049..38457a69 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -1,26 +1,18 @@
 package edu.mit.streamjit.tuner;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-import com.google.common.base.Splitter;
-
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
-import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
-import edu.mit.streamjit.test.Datasets;
 import edu.mit.streamjit.tuner.ConfigGenerator.sqliteAdapter;
 
 /**
@@ -117,63 +109,4 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 				"UPDATE results SET Exectime=%f WHERE Round=%d", time, round);
 		sjDb.executeUpdate(qry);
 	}
-
-	private static String getConfigurationString(String s) {
-		String s1 = s.replaceAll("__class__", "ttttt");
-		String s2 = s1.replaceAll("javaClassPath", "class");
-		String s3 = s2.replaceAll("ttttt", "__class__");
-		return s3;
-	}
-
-	/**
-	 * Creates a new {@link Configuration} from the received python dictionary
-	 * string. This is not a good way to do.
-	 * <p>
-	 * TODO: Need to add a method to {@link Configuration} so that the
-	 * configuration object can be updated from the python dict string. Now we
-	 * are destructing the old confg object and recreating a new one every time.
-	 * Not a appreciatable way.
-	 * 
-	 * @param pythonDict
-	 *            Python dictionary string. Autotuner gives a dictionary of
-	 *            features with trial values.
-	 * @param config
-	 *            Old configuration object.
-	 * @return New configuration object with updated values from the pythonDict.
-	 */
-	private static Configuration rebuildConfiguration(String pythonDict,
-			Configuration config) {
-		// System.out.println(pythonDict);
-		checkNotNull(pythonDict, "Received Python dictionary is null");
-		pythonDict = pythonDict.replaceAll("u'", "");
-		pythonDict = pythonDict.replaceAll("':", "");
-		pythonDict = pythonDict.replaceAll("\\{", "");
-		pythonDict = pythonDict.replaceAll("\\}", "");
-		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
-				.trimResults();
-		Configuration.Builder builder = Configuration.builder();
-		System.out.println("New parameter values from Opentuner...");
-		for (String s : dictSplitter.split(pythonDict)) {
-			String[] str = s.split(" ");
-			if (str.length != 2)
-				throw new AssertionError("Wrong python dictionary...");
-			Parameter p = config.getParameter(str[0]);
-			if (p == null)
-				continue;
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				builder.addParameter(new IntParameter(ip.getName(),
-						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
-
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter sp = (SwitchParameter) p;
-				Class<?> type = sp.getGenericParameter();
-				int val = Integer.parseInt(str[1]);
-				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
-						type, sp.getUniverse().get(val), sp.getUniverse());
-				builder.addParameter(sp1);
-			}
-		}
-		return builder.build();
-	}
 }

From 43628fbba89922a2904ea20c1837d2ad494d913b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 01:59:34 +0800
Subject: [PATCH 116/881] Bug fix

commandStr -> jvmArgs
---
 lib/opentuner/streamjit/tuner2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 97ffbf65..239b086a 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -105,7 +105,7 @@ def runApp(self, cfg_data):
 			print "\033[31;1mException Found\033[0m"
 			self.waitForStreamNodes(True)
 			cur = self.tunedataDB.cursor()
-			str1 = str(commandStr)
+			str1 = str(jvmArgs)
 			str2 = self.config.toJSON()
 			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
 			self.tunedataDB.commit()

From cf6a0ae01fb93e97655d882780e788158c605e34 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:13:50 +0800
Subject: [PATCH 117/881] Saves Json of the final configuration

---
 lib/opentuner/streamjit/tuner2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 239b086a..376c9fa8 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -139,12 +139,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = dict.copy(configuration.data)
+		cfg_data = dict.copy(configuration.data)
+
 		print "\033[32;1mFinal Config...\033[0m"
-		(state, time) = self.runApp(cfg)
+		(state, time) = self.runApp(cfg_data)
 		conn = sqlite3.connect('streamjit.db', 100)
 		cur = conn.cursor()
-		query = 'INSERT INTO FinalResult VALUES ("%s","%s", %d, "%s", "%f")'%(self.program, cfg, self.trycount, state, float(time))
+		query = "INSERT INTO FinalResult VALUES ('%s','%s', %d, '%s', '%f')"%(self.program, self.config.toJSON(), self.trycount, state, float(time))
 		cur.execute(query)
 		conn.commit()
 

From 68816af7f6f01ff1cf346f6bc3df81304568d2ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 02:39:44 +0800
Subject: [PATCH 118/881] Uses compiler2

---
 src/edu/mit/streamjit/tuner/ConfigGenerator.java | 3 +--
 src/edu/mit/streamjit/tuner/RunApp.java          | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 414b2a50..1d5d1401 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.test.Benchmark;
@@ -198,7 +197,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		BlobFactory bf;
 		if (noOfmachines == 1)
-			bf = new CompilerBlobFactory();
+			bf = new Compiler2BlobFactory();
 		else
 			bf = new DistributedBlobFactory(noOfmachines);
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 38457a69..e0ca0db6 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -9,7 +9,7 @@
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
-import edu.mit.streamjit.impl.compiler.CompilerStreamCompiler;
+import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
@@ -65,8 +65,8 @@ public static void main(String[] args) throws SQLException {
 		StreamCompiler sc;
 		IntParameter p = cfg.getParameter("noOfMachines", IntParameter.class);
 		if (p == null) {
-			CompilerStreamCompiler csc = new CompilerStreamCompiler();
-			csc.setConfig(cfg);
+			Compiler2StreamCompiler csc = new Compiler2StreamCompiler();
+			csc.configuration(cfg);
 			sc = csc;
 		} else {
 			sc = new DistributedStreamCompiler(p.getValue(), cfg);

From dcc3c4be16e1c2d3057857376110303b3045aa11 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 09:58:52 +0800
Subject: [PATCH 119/881] Refactored

Methods getArgs1() and getArgs2() have been added.
---
 lib/opentuner/streamjit/tuner2.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/opentuner/streamjit/tuner2.py b/lib/opentuner/streamjit/tuner2.py
index 376c9fa8..63cf021e 100644
--- a/lib/opentuner/streamjit/tuner2.py
+++ b/lib/opentuner/streamjit/tuner2.py
@@ -60,18 +60,15 @@ def runApp(self, cfg_data):
 		#TODO: find a better place for these system-specific constants
 		#the path to the Java executable, or "java" to use system's default
 		javaPath = "java"
-		#the classpath, suitable as the value of the '-cp' java argument
-		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.6.2-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
 
-		args = [javaPath, "-cp", javaClassPath]
 		jvmArgs = []
 		for key in self.jvmOptions.keys():
 			self.jvmOptions.get(key).setValue(cfg_data[key])
 			cmd = self.jvmOptions.get(key).getCommand()
 			if len(cmd) > 0:
 				jvmArgs.append(cmd)
-		args.extend(jvmArgs)
-		args.append("edu.mit.streamjit.tuner.RunApp")
+
+		args = self.getArgs1(javaPath, jvmArgs)
 		args.append(str(self.program))
 		args.append(str(self.trycount))
 
@@ -125,6 +122,22 @@ def runApp(self, cfg_data):
 			self.waitForStreamNodes(False)
 			return ('OK',exetime)
 
+	# Return args that is to run a runnable jar file.
+	def getArgs1(self, javaPath, jvmArgs):
+		args = [javaPath]
+		args.extend(jvmArgs)
+		args.append("-jar")
+		args.append("RunApp.jar")
+		return args
+
+	# Return args that is to run from class file.
+	def getArgs2(self, javaPath, jvmArgs):
+		#the classpath, suitable as the value of the '-cp' java argument
+		javaClassPath = "dist/jstreamit.jar:lib/ASM/asm-debug-all-4.1.jar:lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar:lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar:lib/Guava/guava-15.0.jar:lib/Guava/guava-15.0-javadoc.jar:lib/Guava/guava-15.0-sources.jar:lib/JOptSimple/jopt-simple-4.5.jar:lib/JOptSimple/jopt-simple-4.5-javadoc.jar:lib/JOptSimple/jopt-simple-4.5-sources.jar:lib/jsonp/javax.json-1.0-fab.jar:lib/jsonp/javax.json-api-1.0-SNAPSHOT-javadoc.jar:lib/ServiceProviderProcessor/ServiceProviderProcessor.jar:lib/sqlite/sqlite-jdbc-3.7.15-M1.jar"
+		args = [javaPath, "-cp", javaClassPath]
+		args.append("edu.mit.streamjit.tuner.RunApp")
+		return args
+
 	def niceprint(self, cfg_data):
 		print "\n--------------------------------------------------"
 		print self.trycount

From dc6f69e9ddafdbb0124d1140b25ac6f0513ed748 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 30 Jan 2014 10:03:02 +0800
Subject: [PATCH 120/881] Updated to reflect latest changes

---
 doc/How to tune.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/How to tune.txt b/doc/How to tune.txt
index 34d0d0b6..ff247917 100644
--- a/doc/How to tune.txt	
+++ b/doc/How to tune.txt	
@@ -2,7 +2,7 @@ Go to lib/opentuner/ folder and run venv-bootstrap.py. This will setup a virtual
 
 Run the edu.mit.streamjit.tuner.ConfigGenerator.java to generate configuration information for a particular app. Tuner.java will update the apps table in the streamjit.db with name, configuration, location and class name of the streamjit app.
 
-Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder. If you need to change the input size, you can change it inside the function runApp() in the RunApp class.
+Build the edu.mit.streamjit.tuner.RunApp.java and export it as runnable jar in to stramjit folder.
 
 Run lib/opentuner/streamjit/tuner2.py.
 

From 72fd3167f702516f60f9d616672f34586a1fd1b3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 10:17:36 +0800
Subject: [PATCH 121/881] More organized use of Configuration &  BlobFactory

StreamNodes uses the blobfactory which comes from Configuration.BlobSpecifier.
By fliping GlobalConstants.useCompilerBob flag, we can easily switch between
Interpreter blob and CompilerBlob.
---
 .../distributed/ConfigurationManager.java     | 35 ++++++++++++++++---
 .../distributed/DistributedBlobFactory.java   |  6 ++--
 .../distributed/common/GlobalConstants.java   |  5 +++
 .../node/CfgStringProcessorImpl.java          | 15 ++------
 .../impl/distributed/runtimer/Controller.java | 18 +---------
 5 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 21fe572f..67475cf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -19,9 +20,13 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
@@ -125,17 +130,21 @@ public Configuration getDynamicConfiguration() {
 			PartitionParameter.Builder partParam = PartitionParameter.builder(
 					GlobalConstants.PARTITION, coresPerMachine);
 
-			BlobFactory factory = new Interpreter.InterpreterBlobFactory();
-			partParam.addBlobFactory(factory);
-
+			BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
+			BlobFactory comp2Factory = new Compiler2BlobFactory();
+			partParam.addBlobFactory(intFactory);
+			partParam.addBlobFactory(comp2Factory);
 			app.blobtoMachineMap = new HashMap<>();
 
+			BlobFactory bf = GlobalConstants.useCompilerBlob
+					? comp2Factory
+					: intFactory;
 			for (Integer machineID : app.partitionsMachineMap.keySet()) {
 				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
 						.get(machineID);
 				for (Set<Worker<?, ?>> blobWorkers : blobList) {
 					// TODO: One core per blob. Need to change this.
-					partParam.addBlob(machineID, 1, factory, blobWorkers);
+					partParam.addBlob(machineID, 1, bf, blobWorkers);
 
 					// TODO: Temp fix to build.
 					Token t = Utils.getblobID(blobWorkers);
@@ -144,9 +153,25 @@ public Configuration getDynamicConfiguration() {
 			}
 
 			builder.addParameter(partParam.build());
-			if (app.blobConfiguration != null)
+			if (GlobalConstants.useCompilerBlob)
 				builder.addSubconfiguration("blobConfigs",
 						app.blobConfiguration);
+			else
+				builder.addSubconfiguration("blobConfigs",
+						getInterpreterConfg());
+			return builder.build();
+		}
+
+		private Configuration getInterpreterConfg() {
+			Configuration.Builder builder = Configuration.builder();
+			// TODO: Ensure the need of this switch parameter.
+			List<ChannelFactory> universe = Arrays
+					.<ChannelFactory> asList(new ConcurrentChannelFactory());
+			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
+					"channelFactory", ChannelFactory.class, universe.get(0),
+					universe);
+
+			builder.addParameter(cfParameter);
 			return builder.build();
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 8ded5576..851d15aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -9,6 +9,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
 /**
@@ -61,13 +62,14 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		Configuration distCfg = cfgManager.getDefaultConfiguration(workers,
 				noOfMachines);
-		Configuration.Builder builder = Configuration.builder(distCfg);
+		if (!GlobalConstants.useCompilerBlob)
+			return distCfg;
 
+		Configuration.Builder builder = Configuration.builder(distCfg);
 		BlobFactory compilerBf = new Compiler2BlobFactory();
 		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
-
 		return builder.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 4215f21b..ea7c44ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -105,6 +105,11 @@ private GlobalConstants() {
 	 */
 	public static final int outputCount = 100000;
 
+	/**
+	 * if true uses Compiler2, interpreter otherwise.
+	 */
+	public static final boolean useCompilerBlob = true;
+
 	static {
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index c937d94e..df029b2f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -25,14 +25,12 @@
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter.BlobSpecifier;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
-import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -128,32 +126,25 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			if (blobList == null)
 				return blobSet.build();
 
-			BlobFactory bf;
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
-			if (blobConfigs == null) {
-				blobConfigs = staticConfig;
-				bf = new Interpreter.InterpreterBlobFactory();
-			} else {
-				bf = new Compiler2BlobFactory();
-			}
 
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				// DEBUG
+				// DEBUG MSG
 				System.out.println(String.format(
 						"A new blob with workers %s has been created.",
 						workIdentifiers.toString()));
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 
 				try {
+					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;
 				}
 			}
-
 			return blobSet.build();
 		} else
 			return null;
@@ -200,7 +191,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 		URL url;
 		try {
 			url = jarFile.toURI().toURL();
-			URL[] urls = new URL[] { url };
+			URL[] urls = new URL[]{url};
 
 			ClassLoader loader = new URLClassLoader(urls);
 			Class<?> topStreamClass;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 7d42c667..84533dd1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -13,9 +12,7 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
@@ -29,7 +26,6 @@
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
-import edu.mit.streamjit.impl.interp.ChannelFactory;
 
 /**
  * {@link Controller} controls all {@link StreamNode}s in runtime. It has
@@ -126,25 +122,13 @@ public Map<Integer, Integer> getCoreCount() {
 
 	public void newApp(Configuration staticCfg) {
 		Configuration.Builder builder = Configuration.builder(staticCfg);
-
 		Map<Integer, InetAddress> inetMap = new HashMap<>();
 		for (StreamNodeAgent agent : StreamNodeMap.values())
 			inetMap.put(agent.getNodeID(), agent.getAddress());
 
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
-
-		// TODO: Ensure the need of this switch parameter.
-		List<ChannelFactory> universe = Arrays
-				.<ChannelFactory> asList(new ConcurrentChannelFactory());
-		SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
-				"channelFactory", ChannelFactory.class, universe.get(0),
-				universe);
-
-		builder.addParameter(cfParameter).putExtraData(
-				GlobalConstants.INETADDRESS_MAP, inetMap);
-
+		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
 		this.conProvider = new TCPConnectionProvider(controllerNodeID, inetMap);
-
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
 		sendToAll(json);

From 22a046d75544f7bb7adf7ba1c78a826bf64d22e2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 13:10:27 +0800
Subject: [PATCH 122/881] Sends json value of final confg

Final config was sent as a python dict. Now changed it to Json format.
I overlooked it.
---
 lib/opentuner/streamjit/onlinetuner.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/opentuner/streamjit/onlinetuner.py b/lib/opentuner/streamjit/onlinetuner.py
index 84e11b8e..115c971d 100644
--- a/lib/opentuner/streamjit/onlinetuner.py
+++ b/lib/opentuner/streamjit/onlinetuner.py
@@ -60,10 +60,13 @@ def program_version(self):
 
 	def save_final_config(self, configuration):
 		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
-		cfg = configuration.data
-		print "Final configuration", cfg
+		cfg_data = configuration.data
+		print "Final configuration", cfg_data
+		for k in self.config.params:
+			self.config.getParameter(k).update_value_for_json(cfg_data)
+
 		self.connection.sendmsg("Completed")
-		self.connection.sendmsg("%s\n"%cfg)
+		self.connection.sendmsg(self.config.toJSON())
 		self.connection.close()
 		sys.exit(0)
 

From d23a1c8d062bc73605021efd6790aeb98a78fbdc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 31 Jan 2014 17:01:06 +0800
Subject: [PATCH 123/881] Converts pythonDict to Configuration JSON

---
 .../impl/common/ConfigurationEditor.java      | 136 +++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index f615dcb9..d390ceef 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -1,30 +1,41 @@
 package edu.mit.streamjit.impl.common;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
 
+import com.google.common.base.Splitter;
+
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.BlobFactory;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.fmradio.FMRadio;
+import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
+import edu.mit.streamjit.util.json.Jsonifiers;
 
 public class ConfigurationEditor {
 
 	static String name;
 	static int noofwrks;
+
 	/**
 	 * @param args
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		edit(name, noofwrks);
+		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		 edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
+		// convert();
 	}
 
 	private static void generate(OneToOneElement<?, ?> stream) {
@@ -109,4 +120,127 @@ private static void print(String name) {
 			}
 		}
 	}
+
+	private static void convert() {
+		String appName = "ChannelVocoder7Kernel";
+		Configuration cfg = readConfiguration(String.format("%d%s", 1, appName));
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					String.format("%d%s.cfg", 0, appName)));
+			String pythonDict = reader.readLine();
+
+			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
+			saveConfg(finalCfg, 0);
+		} catch (Exception ex) {
+			ex.printStackTrace();
+		}
+	}
+
+	/**
+	 * Creates a new {@link Configuration} from the received python dictionary
+	 * string. This is not a good way to do.
+	 * <p>
+	 * TODO: Need to add a method to {@link Configuration} so that the
+	 * configuration object can be updated from the python dict string. Now we
+	 * are destructing the old confg object and recreating a new one every time.
+	 * Not a appreciatable way.
+	 * 
+	 * @param pythonDict
+	 *            Python dictionary string. Autotuner gives a dictionary of
+	 *            features with trial values.
+	 * @param config
+	 *            Old configuration object.
+	 * @return New configuration object with updated values from the pythonDict.
+	 */
+	private static Configuration rebuildConfiguration(String pythonDict,
+			Configuration config) {
+		// System.out.println(pythonDict);
+		pythonDict = pythonDict.replaceAll("u'", "");
+		pythonDict = pythonDict.replaceAll("':", "");
+		pythonDict = pythonDict.replaceAll("\\{", "");
+		pythonDict = pythonDict.replaceAll("\\}", "");
+		Splitter dictSplitter = Splitter.on(", ").omitEmptyStrings()
+				.trimResults();
+		Configuration.Builder builder = Configuration.builder();
+		System.out.println("New parameter values from Opentuner...");
+		for (String s : dictSplitter.split(pythonDict)) {
+			String[] str = s.split(" ");
+			if (str.length != 2)
+				throw new AssertionError("Wrong python dictionary...");
+			Parameter p = config.getParameter(str[0]);
+			if (p == null)
+				continue;
+			// System.out.println(String.format("\t%s = %s", str[0], str[1]));
+			if (p instanceof IntParameter) {
+				IntParameter ip = (IntParameter) p;
+				builder.addParameter(new IntParameter(ip.getName(),
+						ip.getMin(), ip.getMax(), Integer.parseInt(str[1])));
+
+			} else if (p instanceof SwitchParameter<?>) {
+				SwitchParameter sp = (SwitchParameter) p;
+				Class<?> type = sp.getGenericParameter();
+				int val = Integer.parseInt(str[1]);
+				SwitchParameter<?> sp1 = new SwitchParameter(sp.getName(),
+						type, sp.getUniverse().get(val), sp.getUniverse());
+				builder.addParameter(sp1);
+			}
+
+		}
+		return builder.build();
+	}
+
+	/**
+	 * TODO: This method is totally unnecessary if we remove the usage of the
+	 * name "class" in side {@link Configuration}.
+	 * 
+	 * @param cfg
+	 * @return
+	 */
+	private static String getConfigurationString(Configuration cfg) {
+		String s = Jsonifiers.toJson(cfg).toString();
+		String s1 = s.replaceAll("__class__", "ttttt");
+		String s2 = s1.replaceAll("class", "javaClassPath");
+		String s3 = s2.replaceAll("ttttt", "__class__");
+		return s3;
+	}
+
+	/**
+	 * Save the configuration.
+	 */
+	private static void saveConfg(Configuration config, int round) {
+		String json = config.toJson();
+		String name = "erer";
+		try {
+
+			File dir = new File(String.format("configurations%s%s",
+					File.separator, name));
+			if (!dir.exists())
+				if (!dir.mkdirs()) {
+					System.err.println("Make directory failed");
+					return;
+				}
+
+			File file = new File(dir, String.format("%d%s.cfg", round, name));
+			FileWriter writer = new FileWriter(file, false);
+			writer.write(json);
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static Configuration readConfiguration(String simpeName) {
+		String name = String.format("%s.cfg", simpeName);
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
 }

From a51221650541bd46040cbb28d129b0b9520530ab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 08:48:38 +0800
Subject: [PATCH 124/881] tryCount -> round

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index a53e1a9d..3fac55cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -46,7 +46,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
-		int tryCount = 0;
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -73,12 +73,12 @@ public void run() {
 
 				System.out
 						.println("----------------------------------------------");
-				System.out.println(tryCount++);
+				System.out.println(round++);
 
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, tryCount);
+					saveConfg(cfgJson, round);
 
 				try {
 					if (!cfgManager.newConfiguration(config)) {

From 0172c22081055cc29c4460f4707f605648da68d9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 09:20:46 +0800
Subject: [PATCH 125/881] Variable round has been made to global

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 3fac55cc..aefa10f8 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -32,6 +32,7 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
+	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -42,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
+		this.round = 0;
 	}
 
 	@Override
 	public void run() {
-		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,

From bcf54d919cbb974600bc9dcdbf2b8ef3d6f119b6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:36:52 +0800
Subject: [PATCH 126/881] Refactored the run method.

New method reconfigure() has been created.
---
 .../distributed/runtimer/OnlineTuner.java     | 136 ++++++++++--------
 1 file changed, 75 insertions(+), 61 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index aefa10f8..f4c88a2b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
+import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -32,7 +33,6 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
-	private int round;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
@@ -43,11 +43,11 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
-		this.round = 0;
 	}
 
 	@Override
 	public void run() {
+		int round = 0;
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
@@ -59,6 +59,8 @@ public void run() {
 			tuner.writeLine("confg");
 			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
 
+			Pair<Boolean, Long> ret;
+
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
@@ -72,65 +74,12 @@ public void run() {
 					break;
 				}
 
-				System.out
-						.println("----------------------------------------------");
-				System.out.println(round++);
-
-				Configuration config = Configuration.fromJson(cfgJson);
-
-				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, round);
-
-				try {
-					if (!cfgManager.newConfiguration(config)) {
-						tuner.writeLine("-1");
-						continue;
-					}
-
-					if (manager.isRunning()) {
-						boolean state = drainer.startDraining(0);
-						if (!state) {
-							System.err
-									.println("Final drain has already been called. no more tuning.");
-							tuner.writeLine("exit");
-							break;
-						}
-
-						System.err.println("awaitDrainedIntrmdiate");
-						drainer.awaitDrainedIntrmdiate();
-
-						if (GlobalConstants.useDrainData) {
-							System.err.println("awaitDrainData...");
-							drainer.awaitDrainData();
-							DrainData drainData = drainer.getDrainData();
-							app.drainData = drainData;
-						}
-					}
-
-					drainer.setBlobGraph(app.blobGraph);
-					System.err.println("Reconfiguring...");
-					if (manager.reconfigure()) {
-						Stopwatch stopwatch = Stopwatch.createStarted();
-						manager.awaitForFixInput();
-						stopwatch.stop();
-						// TODO: need to check the manager's status before
-						// passing
-						// the time. Exceptions, final drain, etc may causes app
-						// to
-						// stop executing.
-						long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-
-						System.out.println("Execution time is " + time
-								+ " milli seconds");
-						tuner.writeLine(new Double(time).toString());
-					} else {
-						tuner.writeLine("-1");
-						continue;
-					}
-				} catch (Exception ex) {
-					System.err
-							.println("Couldn't compile the stream graph with this configuration");
-					tuner.writeLine("-1");
+				ret = reconfigure(cfgJson, round++);
+				if (ret.first) {
+					tuner.writeLine(new Double(ret.second).toString());
+				} else {
+					tuner.writeLine("exit");
+					break;
 				}
 			}
 
@@ -145,6 +94,71 @@ public void run() {
 		}
 	}
 
+	/**
+	 * @param cfgJson
+	 * @param round
+	 * @return if ret.first == false, then no more tuning. ret.second = running
+	 *         time in milliseconds.
+	 */
+	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+		long time;
+		System.out.println("----------------------------------------------");
+		System.out.println(round);
+		Configuration config = Configuration.fromJson(cfgJson);
+
+		if (GlobalConstants.saveAllConfigurations || round == 0)
+			saveConfg(cfgJson, round);
+
+		try {
+			if (!cfgManager.newConfiguration(config)) {
+				return new Pair<Boolean, Long>(true, -1l);
+			}
+
+			if (manager.isRunning()) {
+				boolean state = drainer.startDraining(0);
+				if (!state) {
+					System.err
+							.println("Final drain has already been called. no more tuning.");
+					return new Pair<Boolean, Long>(false, -1l);
+				}
+
+				System.err.println("awaitDrainedIntrmdiate");
+				drainer.awaitDrainedIntrmdiate();
+
+				if (GlobalConstants.useDrainData) {
+					System.err.println("awaitDrainData...");
+					drainer.awaitDrainData();
+					DrainData drainData = drainer.getDrainData();
+					app.drainData = drainData;
+				}
+			}
+
+			drainer.setBlobGraph(app.blobGraph);
+			System.err.println("Reconfiguring...");
+			if (manager.reconfigure()) {
+				Stopwatch stopwatch = Stopwatch.createStarted();
+				manager.awaitForFixInput();
+				stopwatch.stop();
+				// TODO: need to check the manager's status before
+				// passing
+				// the time. Exceptions, final drain, etc may causes app
+				// to
+				// stop executing.
+				time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+				System.out.println("Execution time is " + time
+						+ " milli seconds");
+			} else {
+				time = -1l;
+			}
+		} catch (Exception ex) {
+			System.err
+					.println("Couldn't compile the stream graph with this configuration");
+			time = -1l;
+		}
+		return new Pair<Boolean, Long>(true, time);
+	}
+
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 

From 2d308c58c71b30d7e26354c8c61680f8f4afe325 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 10:58:16 +0800
Subject: [PATCH 127/881] reconfiguration() takes only confg as arg

---
 .../distributed/runtimer/OnlineTuner.java     | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f4c88a2b..9a29d767 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,6 +1,8 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
@@ -74,7 +76,15 @@ public void run() {
 					break;
 				}
 
-				ret = reconfigure(cfgJson, round++);
+				System.out
+						.println("----------------------------------------------");
+				System.out.println(round++);
+				Configuration config = Configuration.fromJson(cfgJson);
+
+				if (GlobalConstants.saveAllConfigurations)
+					saveConfg(cfgJson, round);
+
+				ret = reconfigure(config);
 				if (ret.first) {
 					tuner.writeLine(new Double(ret.second).toString());
 				} else {
@@ -100,15 +110,8 @@ public void run() {
 	 * @return if ret.first == false, then no more tuning. ret.second = running
 	 *         time in milliseconds.
 	 */
-	private Pair<Boolean, Long> reconfigure(String cfgJson, int round) {
+	private Pair<Boolean, Long> reconfigure(Configuration config) {
 		long time;
-		System.out.println("----------------------------------------------");
-		System.out.println(round);
-		Configuration config = Configuration.fromJson(cfgJson);
-
-		if (GlobalConstants.saveAllConfigurations || round == 0)
-			saveConfg(cfgJson, round);
-
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				return new Pair<Boolean, Long>(true, -1l);

From 666de4f4e9384fce019ebb24da6560798bc06c9c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:19:07 +0800
Subject: [PATCH 128/881] Evaluates the final configuration

---
 .../distributed/runtimer/OnlineTuner.java     | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9a29d767..d352c726 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -171,6 +171,14 @@ private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
+
+		Configuration cfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(cfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
 		if (needTermination) {
 			if (manager.isRunning()) {
 				drainer.startDraining(1);
@@ -232,6 +240,55 @@ private void runForever(String cfgJson) {
 		}
 	}
 
+	/**
+	 * Evaluates a configuration.
+	 * 
+	 * @param cfg
+	 *            configuration that needs to be evaluated
+	 * @param cfgName
+	 *            name of the configuration. This is just for logging purpose.
+	 */
+	private void evaluateConfig(Configuration cfg, String cfgName) {
+		FileWriter writer;
+		double total = 0;
+		int count = 3;
+		try {
+			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
+					true);
+			writer.write("\n----------------------------------------\n");
+			writer.write(String.format("Configuration name = %s\n", cfgName));
+			if (cfg != null) {
+				for (int i = 0; i < count; i++) {
+					Pair<Boolean, Long> ret = reconfigure(cfg);
+					writer.write(ret.second.toString());
+					writer.write('\n');
+					total += ret.second;
+				}
+				double avg = total / count;
+				writer.write(String.format("Average execution time = %f%n\n",
+						avg));
+			} else {
+				writer.write("Null configuration\n");
+			}
+			writer.close();
+		} catch (IOException e1) {
+			e1.printStackTrace();
+		}
+	}
+
+	private Configuration readConfiguration(String name) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
+
 	/**
 	 * Save the configuration.
 	 */

From 38a71aef43d6d501d0cb5bdba416229b43fc072e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:22:41 +0800
Subject: [PATCH 129/881] runForever() takes configuration object

previously it took json string of the configuration
---
 .../impl/distributed/runtimer/OnlineTuner.java    | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d352c726..f0229b7b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -172,8 +172,8 @@ private void handleTermination() throws IOException {
 		System.out.println("Tuning finished");
 		saveConfg(finalConfg, 0);
 
-		Configuration cfg = Configuration.fromJson(finalConfg);
-		evaluateConfig(cfg, "Final configuration");
+		Configuration finalcfg = Configuration.fromJson(finalConfg);
+		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
 				app.name));
@@ -186,19 +186,18 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalConfg);
+			runForever(finalcfg);
 		}
 	}
 
 	/**
-	 * TODO: Just copied from the run method. Code duplication between this
-	 * method and the run() method. Try to avoid duplicate code.
+	 * TODO: Just copied from the reconfigure method. Code duplication between
+	 * this method and the reconfigure() method. Try to avoid duplicate code.
 	 * 
-	 * @param cfgJson
+	 * @param config
 	 */
-	private void runForever(String cfgJson) {
+	private void runForever(Configuration config) {
 		System.out.println("runForever");
-		Configuration config = Configuration.fromJson(cfgJson);
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				System.err.println("Invalid final configuration.");

From 5c934a16d8c4c6efb960a1e6e82ab245019c7270 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 12:36:14 +0800
Subject: [PATCH 130/881] Refactored

The method runForever() has been remove and its functionalities are covered
by reconfigure().
---
 .../distributed/runtimer/OnlineTuner.java     | 52 ++-----------------
 1 file changed, 4 insertions(+), 48 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f0229b7b..ed82eb5a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -186,56 +186,12 @@ private void handleTermination() throws IOException {
 				manager.stop();
 			}
 		} else {
-			runForever(finalcfg);
-		}
-	}
-
-	/**
-	 * TODO: Just copied from the reconfigure method. Code duplication between
-	 * this method and the reconfigure() method. Try to avoid duplicate code.
-	 * 
-	 * @param config
-	 */
-	private void runForever(Configuration config) {
-		System.out.println("runForever");
-		try {
-			if (!cfgManager.newConfiguration(config)) {
-				System.err.println("Invalid final configuration.");
-				return;
-			}
-
-			if (manager.isRunning()) {
-				boolean state = drainer.startDraining(0);
-				if (!state) {
-					System.err
-							.println("Final drain has already been called. no more tuning.");
-					return;
-				}
-
-				System.err.println("awaitDrainedIntrmdiate");
-				drainer.awaitDrainedIntrmdiate();
-
-				if (GlobalConstants.useDrainData) {
-					System.err.println("awaitDrainData...");
-					drainer.awaitDrainData();
-					DrainData drainData = drainer.getDrainData();
-					app.drainData = drainData;
-				}
-
-				drainer.setBlobGraph(app.blobGraph);
-			}
-
-			System.err.println("Reconfiguring...");
-			boolean var = manager.reconfigure();
-			if (var) {
+			Pair<Boolean, Long> ret = reconfigure(finalcfg);
+			if (ret.first && ret.second > 0)
 				System.out
-						.println("Application is running with the final configuration.");
-			} else {
+						.println("Application is running forever with the final configuration.");
+			else
 				System.err.println("Invalid final configuration.");
-			}
-		} catch (Exception ex) {
-			System.err
-					.println("Couldn't compile the stream graph with this configuration");
 		}
 	}
 

From 2ec7730e6f25b3c3da5743d7ecb81e2092db3692 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 3 Feb 2014 18:53:27 +0800
Subject: [PATCH 131/881] Comments added

---
 .../streamjit/impl/distributed/ConfigurationManager.java   | 7 +++----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java    | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 67475cf3..b4ab69c6 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -34,14 +34,14 @@
  * ConfigurationManager deals with {@link Configuration}. Mainly, It does
  * following two tasks.
  * <ol>
- * <li>Generates configuration for with appropriate tuning parameters for
- * tuning.
+ * <li>Generates configuration with appropriate tuning parameters (Based on the
+ * search space design strategy) for tuning.
  * <li>Dispatch the configuration given by the open tuner and make blobs
  * accordingly.
  * </ol>
  * 
  * One can implement this interface to try different search space designs as
- * they want.
+ * he want.
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Jan 16, 2014
@@ -164,7 +164,6 @@ public Configuration getDynamicConfiguration() {
 
 		private Configuration getInterpreterConfg() {
 			Configuration.Builder builder = Configuration.builder();
-			// TODO: Ensure the need of this switch parameter.
 			List<ChannelFactory> universe = Arrays
 					.<ChannelFactory> asList(new ConcurrentChannelFactory());
 			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index fa2e6c3e..c7e21c94 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -82,7 +82,7 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
-			while (++i < 30 && !stopFlag.get()) {
+			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
 					latch.await();

From 37598f420c4fe9aa17f3b978b47202290d7dc9c2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:33:52 +0800
Subject: [PATCH 132/881] Prints status msg

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ed82eb5a..4374b466 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -204,6 +204,7 @@ private void handleTermination() throws IOException {
 	 *            name of the configuration. This is just for logging purpose.
 	 */
 	private void evaluateConfig(Configuration cfg, String cfgName) {
+		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
 		int count = 3;

From e3c51a76bba6c84fe1f1fd5cb0d3bc22d79b6218 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 4 Feb 2014 21:34:40 +0800
Subject: [PATCH 133/881] Print msgs have been commented out

---
 .../impl/distributed/HotSpotTuning.java       | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 4b8a6154..deff1201 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -44,16 +44,16 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	@Override
 	public boolean newConfiguration(Configuration config) {
 
-		for (Parameter p : config.getParametersMap().values()) {
-			if (p instanceof IntParameter) {
-				IntParameter ip = (IntParameter) p;
-				System.out.println(ip.getName() + " - " + ip.getValue());
-			} else if (p instanceof SwitchParameter<?>) {
-				SwitchParameter<?> sp = (SwitchParameter<?>) p;
-				System.out.println(sp.getName() + " - " + sp.getValue());
-			} else
-				System.out.println(p.getName() + " - Unknown type");
-		}
+		// for (Parameter p : config.getParametersMap().values()) {
+		// if (p instanceof IntParameter) {
+		// IntParameter ip = (IntParameter) p;
+		// System.out.println(ip.getName() + " - " + ip.getValue());
+		// } else if (p instanceof SwitchParameter<?>) {
+		// SwitchParameter<?> sp = (SwitchParameter<?>) p;
+		// System.out.println(sp.getName() + " - " + sp.getValue());
+		// } else
+		// System.out.println(p.getName() + " - Unknown type");
+		// }
 
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
 		try {
@@ -165,11 +165,10 @@ private class PickHotSpots extends StreamVisitor {
 
 		private Joiner<?, ?> skipJoiner;
 
-		private int minSplitjoinSize = 20;
+		private int minSplitjoinSize = 8;
 
 		/**
-		 * Workers those are going to be part {@link OfflineTuner}
-		 * {@link #currentHotSpot}.
+		 * Workers those are going to be part of {@link #currentHotSpot}.
 		 */
 		List<Worker<?, ?>> workerGropups;
 

From 18e6704b7148b1c4b922b1526aada208323ea516 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:13:48 +0800
Subject: [PATCH 134/881] Comments added

---
 .../streamjit/impl/common/AbstractDrainer.java  | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index c7ae3634..35abf078 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -137,15 +137,14 @@ public final void setBlobGraph(BlobGraph blobGraph) {
 	 * Initiate the draining of the blobgraph. Three type of draining could be
 	 * carried out.
 	 * <ol>
-	 * <li>type 0 - Intermediate draining: In this case, no data from input
-	 * buffer will be consumed and StreamJit app will not be stopped. Rather,
-	 * StreamJit app will be just paused for reconfiguration purpose. This
-	 * draining may be triggered by {@link OnlineTuner}.</li>
-	 * <li>type 1 - Semi final draining: In this case, no data from input buffer
-	 * will be consumed but StreamJit app will be stopped. i.e, StreamJit app
-	 * will be stopped safely without consuming any new input. This draining may
-	 * be triggered by {@link OnlineTuner} after opentuner finish tuning and
-	 * send it's final configuration.</li>
+	 * <li>type 0 - Intermediate draining: In this case, no new data from
+	 * {@link Input} will be consumed and StreamJit app will not be stopped.
+	 * Rather, StreamJit app will be just paused for reconfiguration purpose.
+	 * This draining may be triggered by {@link OnlineTuner}.</li>
+	 * <li>type 1 - Semi final draining: In this case, StreamJit app will be
+	 * stopped safely without consuming any new data from {@link Input}. This
+	 * draining may be triggered by {@link OnlineTuner} after opentuner finish
+	 * tuning and send it's final configuration.</li>
 	 * <li>type 2 - Final draining: At the end of input data. After this
 	 * draining StreamJit app will stop. This draining may be triggered by a
 	 * {@link Input} when it run out of input data.</li>

From 12fb9a9fac5dd8ac954e75ca98cc11e3b0cc8a1d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:15:34 +0800
Subject: [PATCH 135/881] Format changes

---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index d390ceef..2830d72f 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -32,8 +32,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		 generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		 edit(name, noofwrks);
+		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
+		edit(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}

From 619eb3c97349dbf8b27255f39ebc98f8161e6a39 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 14:16:28 +0800
Subject: [PATCH 136/881] Deadlock situation at final drain has been handled

Whenever GlobalConstants.useDrainData is false, irrespective of reqDrainData,
stop(3) will be called. This avoids deadlock.
---
 .../impl/distributed/node/BlobsManagerImpl.java      | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2f3ceca8..37d72161 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -298,10 +298,14 @@ private void doDrain(boolean reqDrainData) {
 			drainState = 1;
 
 			for (BoundaryInputChannel bc : inputChannels.values()) {
-				if (!this.reqDrainData)
-					bc.stop(1);
-				else if (GlobalConstants.useDrainData)
-					bc.stop(2);
+				// TODO: [2014-02-05] rearranged this order to call stop(3)
+				// whenever GlobalConstants.useDrainData is false irrespective
+				// of reqDrainData.
+				if (GlobalConstants.useDrainData)
+					if (!this.reqDrainData)
+						bc.stop(1);
+					else
+						bc.stop(2);
 				else
 					bc.stop(3);
 			}

From 112c297e054fbc9c30ae25eb01f0fe63fa3df1d4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Feb 2014 22:16:07 +0800
Subject: [PATCH 137/881] Filename changed

"hand_" prefix has been added to hand tuned configuration file generation.
---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 2830d72f..4f6005f6 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -51,7 +51,7 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("%s.cfg", stream.getClass().getSimpleName());
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
 
 		try {
 			FileWriter writer = new FileWriter(name, false);

From 57c6aff9c5b29a4895509f7780a83d07d666e1cf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 6 Feb 2014 00:02:16 +0800
Subject: [PATCH 138/881] New methods to deal with HotSpotTuning searchspace

Generates and edits the configuration files those are created by
HotSpotTuning ConfigurationManager.
---
 .../impl/common/ConfigurationEditor.java      | 90 ++++++++++++++++++-
 1 file changed, 87 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 4f6005f6..52518ccd 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,8 +16,12 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
+import edu.mit.streamjit.impl.distributed.HotSpotTuning;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
 import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
 import edu.mit.streamjit.util.json.Jsonifiers;
@@ -32,8 +36,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		generate(new ChannelVocoder7.ChannelVocoder7Kernel());
-		edit(name, noofwrks);
+		generate1(new FilterBank6.FilterBankPipeline());
+		edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}
@@ -61,7 +65,6 @@ private static void generate(OneToOneElement<?, ?> stream) {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
-
 	}
 
 	private static void edit(String name, int maxWor)
@@ -99,6 +102,87 @@ private static void edit(String name, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
+	private static void generate1(OneToOneElement<?, ?> stream) {
+		int noOfnodes = 4;
+
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
+		Worker<?, ?> sink = (Worker<?, ?>) primitiveConnector.getSink();
+		noofwrks = Workers.getIdentifier(sink) + 1;
+
+		StreamJitApp app = new StreamJitApp(stream, source, sink);
+		ConfigurationManager cfgManager = new HotSpotTuning(app);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, noOfnodes);
+
+		Configuration cfg = bf.getDefaultConfiguration(Workers
+				.getAllWorkersInGraph(source));
+
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+
+		try {
+			FileWriter writer = new FileWriter(name, false);
+			writer.write(cfg.toJson());
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static void edit1(String name, int maxWor)
+			throws NumberFormatException, IOException {
+		Configuration cfg;
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			cfg = Configuration.fromJson(json);
+			reader.close();
+		} catch (Exception ex) {
+			System.err.println("File reader error");
+			return;
+		}
+
+		Configuration.Builder builder = Configuration.builder(cfg);
+		BufferedReader keyinreader = new BufferedReader(new InputStreamReader(
+				System.in));
+
+		for (int i = 0; i < maxWor; i++) {
+			String wrkrMachineName = String.format("worker%dtomachine", i);
+			String wrkrCutname = String.format("worker%dcut", i);
+
+			SwitchParameter<Integer> wrkrMachine = cfg.getParameter(
+					wrkrMachineName, SwitchParameter.class);
+			IntParameter wrkrCut = cfg.getParameter(wrkrCutname,
+					IntParameter.class);
+
+			if (wrkrMachine != null) {
+				System.out.println(wrkrMachine.getName() + " - "
+						+ wrkrMachine.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrMachine.getName());
+				builder.addParameter(new SwitchParameter<Integer>(wrkrMachine
+						.getName(), Integer.class, val, wrkrMachine
+						.getUniverse()));
+			}
+
+			if (wrkrCut != null) {
+				System.out.println(wrkrCut.getName() + " - "
+						+ wrkrCut.getValue());
+				int val = Integer.parseInt(keyinreader.readLine());
+				builder.removeParameter(wrkrCut.getName());
+				builder.addParameter(new IntParameter(wrkrCut.getName(),
+						wrkrCut.getRange(), val));
+			}
+		}
+
+		cfg = builder.build();
+		FileWriter writer = new FileWriter(name);
+		writer.write(cfg.toJson());
+		writer.close();
+		System.out.println("Successfully updated");
+	}
+
 	private static void print(String name) {
 		Configuration cfg;
 		try {

From 8c12b1c0d10147e5b8a82491554f12e871a37ea8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:12:54 +0800
Subject: [PATCH 139/881] GlobalConstants.tune made to int

Third running mode, evaluates two configuration files mode, has been added.
Previous two are tune and uses cfg file.
---
 .../impl/distributed/DistributedStreamCompiler.java      | 7 ++-----
 src/edu/mit/streamjit/impl/distributed/TailChannel.java  | 2 +-
 .../impl/distributed/common/GlobalConstants.java         | 9 ++++++++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 25f3e799..d72c22aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,11 +129,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune) {
-
-		} else {
+		if (GlobalConstants.tune == 0)
 			this.cfg = readConfiguration(stream.getClass().getSimpleName());
-		}
 
 		if (cfg == null) {
 			System.err
@@ -200,7 +197,7 @@ public void drain() {
 		manager.reconfigure();
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
-		if (GlobalConstants.tune && this.cfg != null) {
+		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
 					cfgManager, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index c7e21c94..13339177 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -31,7 +31,7 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
-		if (!GlobalConstants.tune) {
+		if (GlobalConstants.tune == 0) {
 			pLogger = new performanceLogger();
 			pLogger.start();
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index ea7c44ac..ffd35ad5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -90,8 +90,15 @@ private GlobalConstants() {
 	 * to pass fixed number of input will be measured for 30 rounds and logged
 	 * into FixedOutPut.txt. See {@link TailChannel} for the file logging
 	 * details.
+	 * <ol>
+	 * 0 - No tuning, uses configuration file to run.
+	 * <ol>
+	 * 1 - Tuning.
+	 * <ol>
+	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
+	 * cfg. Both file should be presented in the running directory.
 	 */
-	public static final boolean tune = false;
+	public static final int tune = 0;
 
 	/**
 	 * Save all configurations tired by open tuner in to

From 466e1340e35aeee3052b04f4c83893fcce0172ea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Feb 2014 08:26:11 +0800
Subject: [PATCH 140/881] Based on the settings, it can evaluate two cfgs.

---
 .../distributed/runtimer/OnlineTuner.java     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4374b466..81688385 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -49,6 +49,16 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
+		if (GlobalConstants.tune == 1)
+			tune();
+		else if (GlobalConstants.tune == 2)
+			evaluate();
+		else
+			System.err
+					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
+	}
+
+	private void tune() {
 		int round = 0;
 		try {
 			tuner.startTuner(String.format(
@@ -104,6 +114,22 @@ public void run() {
 		}
 	}
 
+	private void evaluate() {
+		Configuration finalCfg = readConfiguration(String.format(
+				"final_%s.cfg", app.name));
+		evaluateConfig(finalCfg, "Final configuration");
+
+		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
+				app.name));
+		evaluateConfig(handCfg, "Handtuned configuration");
+
+		if (manager.isRunning()) {
+			drainer.startDraining(1);
+		} else {
+			manager.stop();
+		}
+	}
+
 	/**
 	 * @param cfgJson
 	 * @param round

From 3f6dda3387ae50c0c3e0e0845793342343b7091f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Feb 2014 14:07:43 +0800
Subject: [PATCH 141/881] ignores first run at final meassurement

First run often shows noise. So we ignore it from the average runtime
calculation.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 81688385..a807a80c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -233,15 +233,18 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
-		int count = 3;
+		int count = 8;
 		try {
 			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
 					true);
 			writer.write("\n----------------------------------------\n");
 			writer.write(String.format("Configuration name = %s\n", cfgName));
 			if (cfg != null) {
+				Pair<Boolean, Long> ret = reconfigure(cfg); // often the first
+															// run shows huge
+															// noise.
 				for (int i = 0; i < count; i++) {
-					Pair<Boolean, Long> ret = reconfigure(cfg);
+					ret = reconfigure(cfg);
 					writer.write(ret.second.toString());
 					writer.write('\n');
 					total += ret.second;

From 0d4b31f35cfedd0bf29ca3a6e8e274f0fa40399d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 09:35:20 +0800
Subject: [PATCH 142/881] Verifies the configuration which comes from file

Verifies whether the configuration read from file and the default configuration
have same set of parameters. If the parameter set doesn't match, quit
the execution.
---
 .../distributed/DistributedStreamCompiler.java     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index d72c22aa..4e5970fb 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -129,8 +129,18 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		if (GlobalConstants.tune == 0)
-			this.cfg = readConfiguration(stream.getClass().getSimpleName());
+		if (GlobalConstants.tune == 0) {
+			Configuration cfg1 = readConfiguration(stream.getClass()
+					.getSimpleName());
+			if (!this.cfg.getParametersMap().keySet()
+					.equals(cfg1.getParametersMap().keySet())) {
+				System.err
+						.println("Reading the configuration from configuration file");
+				System.err
+						.println("No matching between parameters in the read configuration and parameters in the default configuration");
+				return null;
+			}
+		}
 
 		if (cfg == null) {
 			System.err

From 3c32c88e8f47a5437b46b13e62709c3744ceee8a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 11:03:06 +0800
Subject: [PATCH 143/881] Integer overflow problem handled

When calculating lcm for buffer sizes, integer overflow occurs. Now all
calculations are done in long.
---
 .../impl/distributed/node/BlobsManagerImpl.java       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 37d72161..e49df5f4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = lcm(minInputBufCapaciy.get(t),
+			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}
@@ -160,7 +160,7 @@ private void addBuffer(Token t, int minSize,
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
-	private int gcd(int a, int b) {
+	private long gcd(long a, long b) {
 		while (true) {
 			if (a == 0)
 				return b;
@@ -171,9 +171,10 @@ private int gcd(int a, int b) {
 		}
 	}
 
-	private int lcm(int a, int b) {
-		int val = gcd(a, b);
-		return val != 0 ? ((a * b) / val) : 0;
+	private long lcm(long a, long b) {
+		long val = gcd(a, b);
+		long quotient = a / val;
+		return val != 0 ? b * quotient : 0;
 	}
 
 	private Set<Token> getLocalTokens(Set<Blob> blobSet) {

From b2dbcb9237d829e5c8d9509063edfa0b8c83cfea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Feb 2014 14:52:56 +0800
Subject: [PATCH 144/881] lcm -> max

buffer is created with max of input and output buffer sizes not with
lcm of those.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e49df5f4..c177eb71 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -129,7 +129,7 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
-			int bufSize = (int) lcm(minInputBufCapaciy.get(t),
+			int bufSize = Math.max(minInputBufCapaciy.get(t),
 					minOutputBufCapaciy.get(t));
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}

From 7a6d16e3eb5cd8b9f0708a22a267053b2ece4693 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Feb 2014 08:11:55 +0800
Subject: [PATCH 145/881] Bug fix

assigning the read configuration to this.cfg.
---
 .../streamjit/impl/distributed/DistributedStreamCompiler.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 4e5970fb..39a8bf48 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -140,6 +140,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 						.println("No matching between parameters in the read configuration and parameters in the default configuration");
 				return null;
 			}
+			this.cfg = cfg1;
 		}
 
 		if (cfg == null) {

From 73dcfd0c4fca05b372d033dd80c26403a65fdd6a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Feb 2014 10:43:00 +0800
Subject: [PATCH 146/881] Just print msgs added

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 +++
 .../impl/distributed/node/CfgStringProcessorImpl.java          | 1 +
 2 files changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index c177eb71..07d07168 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -157,6 +157,7 @@ private void addBuffer(Token t, int minSize,
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder) {
 		// TODO: Just to increase the performance. Change it later
 		int bufSize = Math.max(1000, minSize);
+		System.out.println("Buffer size of " + t.toString() + " is " + bufSize);
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
@@ -274,6 +275,8 @@ private void start() {
 
 			for (Thread t : blobThreads)
 				t.start();
+
+			System.out.println(blobID + " started");
 		}
 
 		private void stop() {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index df029b2f..692e82dd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -145,6 +145,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					return null;
 				}
 			}
+			System.out.println("All blobs have been created");
 			return blobSet.build();
 		} else
 			return null;

From 47ffd8d8bb807d117e00abcfcfdc48cd081dea30 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 26 Feb 2014 08:30:20 +0800
Subject: [PATCH 147/881] Renaming: debugPrint -> debugLevel

---
 .../impl/distributed/HeadChannel.java         |  4 ++--
 .../impl/distributed/TailChannel.java         |  4 ++--
 .../distributed/node/TCPInputChannel.java     | 20 +++++++++----------
 .../distributed/node/TCPOutputChannel.java    | 20 +++++++++----------
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 0061c2ad..698f029b 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -19,8 +19,8 @@
 public class HeadChannel extends TCPOutputChannel {
 
 	public HeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugPrint);
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 	}
 
 	protected void fillUnprocessedData() {
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 13339177..dc6b197b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -25,9 +25,9 @@ public class TailChannel extends TCPInputChannel {
 	private performanceLogger pLogger = null;
 
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint,
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int limit) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugPrint);
+		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 		this.limit = limit;
 		count = 0;
 		latch = new CountDownLatch(1);
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index e3b37734..b6c024d2 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -36,7 +36,7 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private final FileWriter writer;
 
-	private final int debugPrint;
+	private final int debugLevel;
 
 	private final Buffer buffer;
 
@@ -61,12 +61,12 @@ public class TCPInputChannel implements BoundaryInputChannel {
 	private ImmutableList<Object> unProcessedData;
 
 	public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
 		this.name = "TCPInputChannel - " + bufferTokenName;
-		this.debugPrint = debugPrint;
+		this.debugLevel = debugLevel;
 		this.softClosed = false;
 		this.extraBuffer = null;
 		this.unProcessedData = null;
@@ -75,7 +75,7 @@ public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		count = 0;
 
 		FileWriter w = null;
-		if (this.debugPrint == 5) {
+		if (this.debugLevel == 5) {
 			try {
 				w = new FileWriter(name, true);
 				w.write("---------------------------------\n");
@@ -147,7 +147,7 @@ public void receiveData() {
 			Object obj = tcpConnection.readObject();
 			count++;
 
-			if (debugPrint == 3) {
+			if (debugLevel == 3) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ obj.toString());
 			}
@@ -158,7 +158,7 @@ public void receiveData() {
 			}
 
 			while (!this.buffer.write(obj)) {
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName()
 							+ " Buffer FULL - " + obj.toString());
 				}
@@ -186,7 +186,7 @@ public void receiveData() {
 				}
 			}
 
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ count + " no of items have been received");
 			}
@@ -230,7 +230,7 @@ private void finalReceive() {
 				Object obj = tcpConnection.readObject();
 				count++;
 
-				if (debugPrint == 2) {
+				if (debugLevel == 2) {
 					System.out.println(Thread.currentThread().getName()
 							+ " finalReceive - " + obj.toString());
 				}
@@ -243,7 +243,7 @@ private void finalReceive() {
 				hasData = true;
 
 				while (!buffer.write(obj)) {
-					if (debugPrint == 3) {
+					if (debugLevel == 3) {
 						System.out.println(Thread.currentThread().getName()
 								+ " finalReceive:Buffer FULL - "
 								+ obj.toString());
@@ -276,7 +276,7 @@ private void finalReceive() {
 					}
 				}
 
-				if (count % 1000 == 0 && debugPrint == 2) {
+				if (count % 1000 == 0 && debugLevel == 2) {
 					System.out.println(Thread.currentThread().getName() + " - "
 							+ count + " no of items have been received");
 				}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index e9e8d8cf..0a74705e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -29,7 +29,7 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	FileWriter writer;
 
-	private final int debugPrint;
+	private final int debugLevel;
 
 	private final Buffer buffer;
 
@@ -50,19 +50,19 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 	protected ImmutableList<Object> unProcessedData;
 
 	public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugPrint) {
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
 		this.stopFlag = new AtomicBoolean(false);
 		this.isFinal = false;
 		this.name = "TCPOutputChannel - " + bufferTokenName;
-		this.debugPrint = debugPrint;
+		this.debugLevel = debugLevel;
 		this.unProcessedData = null;
 		count = 0;
 
 		FileWriter w = null;
-		if (this.debugPrint == 5) {
+		if (this.debugLevel == 5) {
 			try {
 				w = new FileWriter(name, true);
 				w.write("---------------------------------\n");
@@ -120,7 +120,7 @@ public void run() {
 					}
 				}
 
-				if (debugPrint > 0) {
+				if (debugLevel > 0) {
 					System.err.println(Thread.currentThread().getName()
 							+ " - Exiting...");
 					System.out.println("isFinal " + isFinal);
@@ -137,7 +137,7 @@ public final void sendData() {
 				tcpConnection.writeObject(obj);
 				count++;
 
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName() + " - "
 							+ obj.toString());
 				}
@@ -151,7 +151,7 @@ public final void sendData() {
 						.println("TCP Output Channel. WriteObject exception.");
 				reConnect();
 			}
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName() + " - "
 						+ count + " items have been sent");
 			}
@@ -165,7 +165,7 @@ public final int getOtherNodeID() {
 
 	@Override
 	public final void stop(boolean isFinal) {
-		if (debugPrint > 0)
+		if (debugLevel > 0)
 			System.out.println(Thread.currentThread().getName()
 					+ " - stop request");
 		this.isFinal = isFinal;
@@ -183,7 +183,7 @@ private void finalSend() {
 				tcpConnection.writeObject(o);
 				count++;
 
-				if (debugPrint == 3) {
+				if (debugLevel == 3) {
 					System.out.println(Thread.currentThread().getName()
 							+ " FinalSend - " + o.toString());
 				}
@@ -196,7 +196,7 @@ private void finalSend() {
 			} catch (IOException e) {
 				System.err.println("TCP Output Channel. finalSend exception.");
 			}
-			if (count % 1000 == 0 && debugPrint == 2) {
+			if (count % 1000 == 0 && debugLevel == 2) {
 				System.out.println(Thread.currentThread().getName()
 						+ " FinalSend - " + count
 						+ " no of items have been sent");

From 8cff6f67577aa66f4908cb657077a85806efaa9c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 8 Mar 2014 01:19:06 +0800
Subject: [PATCH 148/881] Just print msg changed

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index a807a80c..869508e4 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -86,9 +86,9 @@ private void tune() {
 					break;
 				}
 
-				System.out
-						.println("----------------------------------------------");
-				System.out.println(round++);
+				System.out.println(String.format(
+						"---------------------%d-------------------------",
+						++round));
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)

From c39b75123fad39ae273168777c4f47054b28f8ee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Mar 2014 11:54:59 +0800
Subject: [PATCH 149/881] Monitoring buffer status feature has been added

A seperate thread prints buffer status timely.

Conflicts:
	src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
	src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
---
 .../impl/distributed/node/BlobsManager.java   |  9 +++
 .../distributed/node/BlobsManagerImpl.java    | 71 +++++++++++++++++++
 .../node/CfgStringProcessorImpl.java          |  4 ++
 3 files changed, 84 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
index a5a78919..74848355 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
@@ -17,4 +17,13 @@ public interface BlobsManager {
 	public CTRLRDrainProcessor getDrainProcessor();
 
 	public CommandProcessor getCommandProcessor();
+
+	/**
+	 * For all final resource cleanup. Mainly all started threads must be
+	 * stopped safely.
+	 * <p>
+	 * TODO: [2014-03-05] I added this as a quick fix to clean up
+	 * {@link BlobsManagerImpl}#MonitorBuffers thread. Revise this.
+	 */
+	public void stop();
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 07d07168..4932ad3c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -5,6 +5,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -48,6 +49,8 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final TCPConnectionProvider conProvider;
 	private Map<Token, TCPConnectionInfo> conInfoMap;
 
+	private MonitorBuffers monBufs;
+
 	private final CTRLRDrainProcessor drainProcessor;
 
 	private final CommandProcessor cmdProcessor;
@@ -89,6 +92,14 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 	public void start() {
 		for (BlobExecuter be : blobExecuters)
 			be.start();
+
+		if (monBufs == null) {
+			System.out.println("Creating new MonitorBuffers");
+			monBufs = new MonitorBuffers();
+			monBufs.start();
+		} else
+			System.err
+					.println("Mon buffer is not null. Check the logic for bug");
 	}
 
 	/**
@@ -98,6 +109,9 @@ public void start() {
 	public void stop() {
 		for (BlobExecuter be : blobExecuters)
 			be.stop();
+
+		if (monBufs != null)
+			monBufs.stopMonitoring();
 	}
 
 	// TODO: Buffer sizes, including head and tail buffers, must be optimized.
@@ -295,6 +309,9 @@ private void stop() {
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
+
+			if (monBufs != null)
+				monBufs.stopMonitoring();
 		}
 
 		private void doDrain(boolean reqDrainData) {
@@ -416,6 +433,17 @@ private void drained() {
 				// System.out.println("**********************************");
 			}
 
+			boolean isLastBlob = true;
+			for (BlobExecuter be : blobExecuters) {
+				if (be.drainState < 4) {
+					isLastBlob = false;
+					break;
+				}
+			}
+
+			if (isLastBlob && monBufs != null)
+				monBufs.stopMonitoring();
+
 			// printDrainedStatus();
 		}
 
@@ -579,4 +607,47 @@ public void processSTOP() {
 			}
 		}
 	}
+
+	private static int count = 0;
+
+	private class MonitorBuffers extends Thread {
+		private final int id;
+		private final AtomicBoolean stopFlag;
+		int sleepTime = 25000;
+		MonitorBuffers() {
+			stopFlag = new AtomicBoolean(false);
+			id = count++;
+		}
+
+		public void run() {
+
+			while (!stopFlag.get()) {
+				System.out.println("********Started*************** - " + id);
+				try {
+					Thread.sleep(sleepTime);
+				} catch (InterruptedException e) {
+				}
+				if (bufferMap == null) {
+					System.out.println("Buffer map is null...");
+					continue;
+				}
+				if (stopFlag.get())
+					break;
+				System.out.println("----------------------------------");
+				for (Map.Entry<Token, Buffer> en : bufferMap.entrySet()) {
+					System.out.println(en.getKey() + " - "
+							+ en.getValue().size());
+				}
+				System.out.println("----------------------------------");
+			}
+
+			System.out.println("********Stopped*************** - " + id);
+		}
+
+		public void stopMonitoring() {
+			System.out.println("MonitorBuffers: Stop monitoring");
+			stopFlag.set(true);
+			this.interrupt();
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 692e82dd..92082f8e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -82,6 +82,10 @@ public void process(String json, ConfigType type, DrainData drainData) {
 				Map<Token, TCPConnectionInfo> conInfoMap = (Map<Token, TCPConnectionInfo>) cfg
 						.getExtraData(GlobalConstants.CONINFOMAP);
 
+				BlobsManager bm = streamNode.getBlobsManager();
+				if (bm != null)
+					bm.stop();
+
 				streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
 						conInfoMap, streamNode, conProvider));
 			} else {

From de4b37f051ebaf9ce92efb16773ebc2a618284ee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 6 Mar 2014 13:40:34 +0800
Subject: [PATCH 150/881] Writes buffer status into a file.

Runtimer writes the all buffer status in to a file time to time.

Conflicts:
	src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
---
 .../distributed/node/BlobsManagerImpl.java    | 58 +++++++++++++------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 4932ad3c..f72bb157 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -620,28 +621,47 @@ private class MonitorBuffers extends Thread {
 		}
 
 		public void run() {
-
-			while (!stopFlag.get()) {
-				System.out.println("********Started*************** - " + id);
-				try {
-					Thread.sleep(sleepTime);
-				} catch (InterruptedException e) {
-				}
-				if (bufferMap == null) {
-					System.out.println("Buffer map is null...");
-					continue;
-				}
-				if (stopFlag.get())
-					break;
-				System.out.println("----------------------------------");
-				for (Map.Entry<Token, Buffer> en : bufferMap.entrySet()) {
-					System.out.println(en.getKey() + " - "
-							+ en.getValue().size());
+			FileWriter writter = null;
+			try {
+				writter = new FileWriter(String.format("BufferStatus%d.txt",
+						streamNode.getNodeID()), false);
+
+				writter.write(String.format(
+						"********Started*************** - %d\n", id));
+				while (!stopFlag.get()) {
+					try {
+						Thread.sleep(sleepTime);
+					} catch (InterruptedException e) {
+					}
+					if (bufferMap == null) {
+						writter.write("Buffer map is null...\n");
+						continue;
+					}
+					if (stopFlag.get())
+						break;
+					writter.write("----------------------------------\n");
+					for (Map.Entry<Token, Buffer> en : bufferMap.entrySet()) {
+						writter.write(en.getKey() + " - "
+								+ en.getValue().size());
+						writter.write('\n');
+					}
+					writter.write("----------------------------------\n");
+					writter.flush();
 				}
-				System.out.println("----------------------------------");
+
+				writter.write(String.format(
+						"********Stopped*************** - %d\n", id));
+			} catch (IOException e1) {
+				e1.printStackTrace();
+				return;
 			}
 
-			System.out.println("********Stopped*************** - " + id);
+			try {
+				if (writter != null)
+					writter.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 		}
 
 		public void stopMonitoring() {

From 12cb1d5d69a01980c37cbb972f79b03da172365b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Mar 2014 14:49:12 +0800
Subject: [PATCH 151/881] GlobalConstants.printOutputCount has been added

Prints number of outputs generated.
---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java    | 3 ++-
 .../streamjit/impl/distributed/common/GlobalConstants.java | 7 ++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index dc6b197b..6d23279b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -41,7 +41,8 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 	public void receiveData() {
 		super.receiveData();
 		count++;
-		// System.err.println(count);
+		if (GlobalConstants.printOutputCount && count % 10000 == 0)
+			System.err.println(count);
 		if (count > limit)
 			latch.countDown();
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index ffd35ad5..2972b196 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -110,13 +110,18 @@ private GlobalConstants() {
 	 * Output count for tuning. Tuner measures the running time for this number
 	 * of outputs.
 	 */
-	public static final int outputCount = 100000;
+	public static final int outputCount = 40000;
 
 	/**
 	 * if true uses Compiler2, interpreter otherwise.
 	 */
 	public static final boolean useCompilerBlob = true;
 
+	/**
+	 * Prints number of outputs generated.
+	 */
+	public static final boolean printOutputCount = false;
+
 	static {
 
 	}

From 6ffa672a920b29e263e6067086cddad87436178a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Mar 2014 11:24:57 +0800
Subject: [PATCH 152/881] Made public and added default constructor.

---
 .../mit/streamjit/test/apps/filterbank6/FilterBank6.java    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java b/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
index b98e0e4f..b57bac23 100644
--- a/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
+++ b/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
@@ -61,11 +61,15 @@ public OneToOneElement<Object, Object> instantiate() {
 	/**
 	 * Top-level filterbank structure.
 	 **/
-	private static final class FilterBankPipeline extends Pipeline<Float, Float> {
+	public static final class FilterBankPipeline extends Pipeline<Float, Float> {
 		private FilterBankPipeline(int M) {
 			add(new FilterBankSplitJoin(M));
 			add(new Adder(M));
 		}
+
+		public FilterBankPipeline() {
+			this(32);
+		}
 	}
 
 	/**

From b6b2201206420d3bb3fbdfd2507abd6198710280 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Mar 2014 22:28:56 +0800
Subject: [PATCH 153/881] Flag singleNodeOnline has been added

By setting GlobalConstants.singleNodeOnline flag, distributed stream compiler
can be used for single node online tuning.
noOfnodes made private final.
---
 .../distributed/DistributedStreamCompiler.java    | 15 +++++++++++----
 .../impl/distributed/common/GlobalConstants.java  | 13 ++++++++++---
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 39a8bf48..06a7a1c8 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -68,12 +68,12 @@ public class DistributedStreamCompiler implements StreamCompiler {
 	/**
 	 * Configuration from Opentuner.
 	 */
-	Configuration cfg;
+	private Configuration cfg;
 
 	/**
 	 * Total number of nodes including controller node.
 	 */
-	int noOfnodes;
+	private int noOfnodes;
 
 	/**
 	 * @param noOfnodes
@@ -118,8 +118,15 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		stream.visit(verifier);
 
 		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
-		// conTypeCount.put(CommunicationType.LOCAL, 1);
-		conTypeCount.put(CommunicationType.TCP, this.noOfnodes);
+
+		if (GlobalConstants.singleNodeOnline) {
+			System.out
+					.println("Flag GlobalConstants.singleNodeOnline is enabled."
+							+ " noOfNodes passed as compiler argument has no effect");
+			conTypeCount.put(CommunicationType.LOCAL, 1);
+			this.noOfnodes = 1;
+		}
+		conTypeCount.put(CommunicationType.TCP, this.noOfnodes - 1);
 		Controller controller = new Controller();
 		controller.connect(conTypeCount);
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 2972b196..9405c5ce 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -1,6 +1,7 @@
 package edu.mit.streamjit.impl.distributed.common;
 
 import edu.mit.streamjit.impl.common.AbstractDrainer;
+import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.TailChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
@@ -98,7 +99,7 @@ private GlobalConstants() {
 	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
 	 * cfg. Both file should be presented in the running directory.
 	 */
-	public static final int tune = 0;
+	public static final int tune = 1;
 
 	/**
 	 * Save all configurations tired by open tuner in to
@@ -110,7 +111,7 @@ private GlobalConstants() {
 	 * Output count for tuning. Tuner measures the running time for this number
 	 * of outputs.
 	 */
-	public static final int outputCount = 40000;
+	public static final int outputCount = 100000;
 
 	/**
 	 * if true uses Compiler2, interpreter otherwise.
@@ -120,7 +121,13 @@ private GlobalConstants() {
 	/**
 	 * Prints number of outputs generated.
 	 */
-	public static final boolean printOutputCount = false;
+	public static final boolean printOutputCount = true;
+
+	/**
+	 * Enables {@link DistributedStreamCompiler} to run on a single node. When
+	 * this is enabled, noOfNodes passed as compiler argument has no effect.
+	 */
+	public static final boolean singleNodeOnline = true;
 
 	static {
 

From 652c0cc590ac06f86f2d8e01829b09ff89fcd8bf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Mar 2014 23:47:07 +0800
Subject: [PATCH 154/881] Skip some outputs to get steady state run time

---
 .../impl/distributed/TailChannel.java         | 54 +++++++++++++------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 6d23279b..28ef9c90 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -16,21 +16,33 @@
 
 public class TailChannel extends TCPInputChannel {
 
-	int limit;
+	private final int steadyCount;
 
-	int count;
+	private final int skipCount;
 
-	private volatile CountDownLatch latch;
+	private final int totalCount;
+
+	private int count;
+
+	private volatile CountDownLatch steadyLatch;
+
+	private volatile CountDownLatch skipLatch;
 
 	private performanceLogger pLogger = null;
 
+	private boolean skipLatchUp;
+
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel,
-			int limit) {
+			int skipCount, int steadyCount) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
-		this.limit = limit;
+		this.steadyCount = steadyCount;
+		this.skipCount = skipCount;
+		this.totalCount = steadyCount + skipCount;
 		count = 0;
-		latch = new CountDownLatch(1);
+		steadyLatch = new CountDownLatch(1);
+		skipLatch = new CountDownLatch(1);
+		this.skipLatchUp = true;
 		if (GlobalConstants.tune == 0) {
 			pLogger = new performanceLogger();
 			pLogger.start();
@@ -41,12 +53,18 @@ public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 	public void receiveData() {
 		super.receiveData();
 		count++;
+
 		if (GlobalConstants.printOutputCount && count % 10000 == 0)
 			System.err.println(count);
-		if (count > limit)
-			latch.countDown();
-	}
 
+		if (skipLatchUp && count > skipCount) {
+			skipLatch.countDown();
+			skipLatchUp = false;
+		}
+
+		if (count > totalCount)
+			steadyLatch.countDown();
+	}
 	@Override
 	public void stop(int type) {
 		super.stop(type);
@@ -56,14 +74,21 @@ public void stop(int type) {
 		}
 	}
 
-	public void awaitForFixInput() throws InterruptedException {
-		latch.await();
+	public long awaitForFixInput() throws InterruptedException {
+		skipLatch.await();
+		Stopwatch stopwatch = Stopwatch.createStarted();
+		steadyLatch.await();
+		stopwatch.stop();
+		return stopwatch.elapsed(TimeUnit.MILLISECONDS);
 	}
 
 	public void reset() {
-		latch.countDown();
-		latch = new CountDownLatch(1);
+		steadyLatch.countDown();
+		steadyLatch = new CountDownLatch(1);
+		skipLatch.countDown();
+		skipLatch = new CountDownLatch(1);
 		count = 0;
+		skipLatchUp = true;
 	}
 
 	private class performanceLogger extends Thread {
@@ -86,7 +111,7 @@ public void run() {
 			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Stopwatch stopwatch = Stopwatch.createStarted();
-					latch.await();
+					steadyLatch.await();
 					stopwatch.stop();
 					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
@@ -105,7 +130,6 @@ public void run() {
 			try {
 				writer.close();
 			} catch (IOException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}

From 6a473b656c630ac34c2fc712c099080e4a8b020b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Mar 2014 23:54:32 +0800
Subject: [PATCH 155/881] TailChannel returns fix out put time

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 4 ++--
 src/edu/mit/streamjit/impl/distributed/TailChannel.java     | 6 +-----
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 5 +----
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 21348536..4c926dea 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -250,8 +250,8 @@ else if (GlobalConstants.useDrainData)
 		isRunning = false;
 	}
 
-	public void awaitForFixInput() throws InterruptedException {
-		tailChannel.awaitForFixInput();
+	public long awaitForFixInput() throws InterruptedException {
+		return tailChannel.awaitForFixInput();
 	}
 
 	public void setDrainer(AbstractDrainer drainer) {
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 28ef9c90..1b032d45 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -110,11 +110,7 @@ public void run() {
 			}
 			while (++i < 10 && !stopFlag.get()) {
 				try {
-					Stopwatch stopwatch = Stopwatch.createStarted();
-					steadyLatch.await();
-					stopwatch.stop();
-					Long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-
+					Long time = awaitForFixInput();
 					System.out.println("Execution time is " + time
 							+ " milli seconds");
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 869508e4..49e62096 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -165,15 +165,12 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			drainer.setBlobGraph(app.blobGraph);
 			System.err.println("Reconfiguring...");
 			if (manager.reconfigure()) {
-				Stopwatch stopwatch = Stopwatch.createStarted();
-				manager.awaitForFixInput();
-				stopwatch.stop();
 				// TODO: need to check the manager's status before
 				// passing
 				// the time. Exceptions, final drain, etc may causes app
 				// to
 				// stop executing.
-				time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+				time = manager.awaitForFixInput();
 
 				System.out.println("Execution time is " + time
 						+ " milli seconds");

From 39faeec948e0938f83d4d72566f1be51691c6253 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Mar 2014 23:57:49 +0800
Subject: [PATCH 156/881] Renamed: awaitFixInput() -> getFixedOutputTime()

---
 .../streamjit/impl/distributed/StreamJitAppManager.java    | 7 ++++---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java    | 4 ++--
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 4c926dea..71f46bc3 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,8 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, GlobalConstants.outputCount);
+						+ tailToken.toString(), 0, 0,
+				GlobalConstants.outputCount);
 	}
 
 	/**
@@ -250,8 +251,8 @@ else if (GlobalConstants.useDrainData)
 		isRunning = false;
 	}
 
-	public long awaitForFixInput() throws InterruptedException {
-		return tailChannel.awaitForFixInput();
+	public long getFixedOutputTime() throws InterruptedException {
+		return tailChannel.getFixedOutputTime();
 	}
 
 	public void setDrainer(AbstractDrainer drainer) {
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 1b032d45..85aa1202 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -74,7 +74,7 @@ public void stop(int type) {
 		}
 	}
 
-	public long awaitForFixInput() throws InterruptedException {
+	public long getFixedOutputTime() throws InterruptedException {
 		skipLatch.await();
 		Stopwatch stopwatch = Stopwatch.createStarted();
 		steadyLatch.await();
@@ -110,7 +110,7 @@ public void run() {
 			}
 			while (++i < 10 && !stopFlag.get()) {
 				try {
-					Long time = awaitForFixInput();
+					Long time = getFixedOutputTime();
 					System.out.println("Execution time is " + time
 							+ " milli seconds");
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 49e62096..baca6047 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -170,7 +170,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				// the time. Exceptions, final drain, etc may causes app
 				// to
 				// stop executing.
-				time = manager.awaitForFixInput();
+				time = manager.getFixedOutputTime();
 
 				System.out.println("Execution time is " + time
 						+ " milli seconds");

From 0febebd559683c06a331d86e94f55c01eb9923f9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 13 Mar 2014 09:01:19 +0800
Subject: [PATCH 157/881] Misc changes

Comments added to TailChannel. Default SkipCount is set to
GlobalConstants.OutputCount.
---
 .../DistributedStreamCompiler.java            |  4 +--
 .../impl/distributed/StreamJitAppManager.java |  2 +-
 .../impl/distributed/TailChannel.java         | 29 +++++++++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 06a7a1c8..3cf75422 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -180,9 +180,9 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		// TODO: derive a algorithm to find good buffer size and use here.
 		Buffer head = InputBufferFactory.unwrap(input).createReadableBuffer(
-				1000);
+				10000);
 		Buffer tail = OutputBufferFactory.unwrap(output).createWritableBuffer(
-				1000);
+				10000);
 
 		boolean needTermination;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 71f46bc3..5869194d 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -176,7 +176,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, 0,
+						+ tailToken.toString(), 0, GlobalConstants.outputCount,
 				GlobalConstants.outputCount);
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 85aa1202..ad94ca25 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -16,8 +16,6 @@
 
 public class TailChannel extends TCPInputChannel {
 
-	private final int steadyCount;
-
 	private final int skipCount;
 
 	private final int totalCount;
@@ -32,11 +30,26 @@ public class TailChannel extends TCPInputChannel {
 
 	private boolean skipLatchUp;
 
+	/**
+	 * @param buffer
+	 * @param conProvider
+	 * @param conInfo
+	 * @param bufferTokenName
+	 * @param debugLevel
+	 * @param skipCount
+	 *            : Skips this amount of output before evaluating the running
+	 *            time. This is added to avoid the noise from init schedule and
+	 *            the drain data. ( i.e., In order to get real steady state
+	 *            execution time)
+	 * @param steadyCount
+	 *            : {@link #getFixedOutputTime()} calculates the time taken to
+	 *            get this amount of outputs ( after skipping skipCount number
+	 *            of outputs at the beginning).
+	 */
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int skipCount, int steadyCount) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
-		this.steadyCount = steadyCount;
 		this.skipCount = skipCount;
 		this.totalCount = steadyCount + skipCount;
 		count = 0;
@@ -65,6 +78,7 @@ public void receiveData() {
 		if (count > totalCount)
 			steadyLatch.countDown();
 	}
+
 	@Override
 	public void stop(int type) {
 		super.stop(type);
@@ -74,6 +88,15 @@ public void stop(int type) {
 		}
 	}
 
+	/**
+	 * Skips skipCount amount of output at the beginning and then calculates the
+	 * time taken to get steadyCount amount of outputs. skipCount is added to
+	 * avoid the noise from init schedule and the drain data. ( i.e., In order
+	 * to get real steady state execution time).
+	 * 
+	 * @return time in MILLISECONDS.
+	 * @throws InterruptedException
+	 */
 	public long getFixedOutputTime() throws InterruptedException {
 		skipLatch.await();
 		Stopwatch stopwatch = Stopwatch.createStarted();

From 5de82fd7915935dfe25cd8009bdeb46563ab2162 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 13 Mar 2014 09:56:54 +0800
Subject: [PATCH 158/881] keeps headToken and tailToken

Instead of calculating these tokens again and agian, class stores those
as final class variables.
---
 .../impl/distributed/StreamJitAppManager.java  | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 5869194d..e9311642 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -52,6 +52,10 @@ public class StreamJitAppManager {
 
 	private final ConfigurationManager cfgManager;
 
+	private final Token headToken;
+
+	private final Token tailToken;
+
 	private boolean isRunning;
 
 	/**
@@ -76,7 +80,7 @@ public class StreamJitAppManager {
 
 	private volatile AppStatus status;
 
-	Map<Token, TCPConnectionInfo> conInfoMap;
+	private Map<Token, TCPConnectionInfo> conInfoMap;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
 			ConfigurationManager cfgManager) {
@@ -93,6 +97,9 @@ public StreamJitAppManager(Controller controller, StreamJitApp app,
 																// good calling
 																// place.
 		isRunning = false;
+
+		headToken = Token.createOverallInputToken(app.source);
+		tailToken = Token.createOverallOutputToken(app.sink);
 	}
 
 	public boolean reconfigure() {
@@ -122,9 +129,7 @@ public boolean reconfigure() {
 			controller.send(nodeID, json);
 		}
 
-		setupHeadTail(conInfoMap, app.bufferMap,
-				Token.createOverallInputToken(app.source),
-				Token.createOverallOutputToken(app.sink));
+		setupHeadTail(conInfoMap, app.bufferMap);
 
 		boolean isCompiled = apStsPro.waitForCompilation();
 
@@ -143,12 +148,9 @@ public boolean reconfigure() {
 	 * 
 	 * @param cfg
 	 * @param bufferMap
-	 * @param headToken
-	 * @param tailToken
 	 */
 	private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
-			ImmutableMap<Token, Buffer> bufferMap, Token headToken,
-			Token tailToken) {
+			ImmutableMap<Token, Buffer> bufferMap) {
 
 		TCPConnectionInfo headconInfo = conInfoMap.get(headToken);
 		assert headconInfo != null : "No head connection info exists in conInfoMap";

From a567725ffc456fec2e3952283f75aff11b93c97b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 13 Mar 2014 16:31:33 +0800
Subject: [PATCH 159/881] if noOfnodes == 1, run everything locally.

DistributedStreamCompiler will run everything locally if either
GlobalConstants.singleNodeOnline is true or noOfNodes passed is 1.
Default constructor also sets noOfNodes to 1.
---
 .../impl/distributed/DistributedStreamCompiler.java        | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 3cf75422..91fddb9a 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -123,10 +123,13 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			System.out
 					.println("Flag GlobalConstants.singleNodeOnline is enabled."
 							+ " noOfNodes passed as compiler argument has no effect");
-			conTypeCount.put(CommunicationType.LOCAL, 1);
 			this.noOfnodes = 1;
 		}
-		conTypeCount.put(CommunicationType.TCP, this.noOfnodes - 1);
+
+		if (this.noOfnodes == 1)
+			conTypeCount.put(CommunicationType.LOCAL, 1);
+		else
+			conTypeCount.put(CommunicationType.TCP, this.noOfnodes - 1);
 		Controller controller = new Controller();
 		controller.connect(conTypeCount);
 

From 2c7dfc399f85965fb1ab47caa116abf72adb11d2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 13 Mar 2014 21:30:18 +0800
Subject: [PATCH 160/881] Bug Fix

When counting for noOfNodes we should include the controller node also.
---
 .../impl/distributed/DistributedStreamCompiler.java          | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 91fddb9a..1790f6fd 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -135,7 +135,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		StreamJitApp app = new StreamJitApp(stream, source, sink);
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, noOfnodes);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, Math.max(
+				noOfnodes - 1, 1));
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
@@ -156,7 +157,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		if (cfg == null) {
 			System.err
 					.println("Configuration is null. Runs the app with horizontal partitioning.");
-			Integer[] machineIds = new Integer[this.noOfnodes];
+			Integer[] machineIds = new Integer[this.noOfnodes - 1];
 			for (int i = 0; i < machineIds.length; i++) {
 				machineIds[i] = i + 1;
 			}

From d99566f21bffb617a2299674766d88c4c17f9ae1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 02:23:29 +0800
Subject: [PATCH 161/881] Uses multiplier to evaluate performance

---
 .../impl/distributed/DistributedStreamCompiler.java    |  2 +-
 .../impl/distributed/StreamJitAppManager.java          | 10 +++++-----
 .../mit/streamjit/impl/distributed/TailChannel.java    |  5 ++++-
 .../impl/distributed/runtimer/OnlineTuner.java         |  7 +++++--
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 1790f6fd..f47a012f 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -216,7 +216,7 @@ public void drain() {
 		app.bufferMap = bufferMapBuilder.build();
 		app.constraints = constraints;
 
-		manager.reconfigure();
+		manager.reconfigure(1);
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
 		if (GlobalConstants.tune > 0 && this.cfg != null) {
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index e9311642..fa59d09d 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -102,7 +102,7 @@ public StreamJitAppManager(Controller controller, StreamJitApp app,
 		tailToken = Token.createOverallOutputToken(app.sink);
 	}
 
-	public boolean reconfigure() {
+	public boolean reconfigure(int multiplier) {
 		reset();
 		Configuration.Builder builder = Configuration.builder(cfgManager
 				.getDynamicConfiguration());
@@ -129,7 +129,7 @@ public boolean reconfigure() {
 			controller.send(nodeID, json);
 		}
 
-		setupHeadTail(conInfoMap, app.bufferMap);
+		setupHeadTail(conInfoMap, app.bufferMap, multiplier);
 
 		boolean isCompiled = apStsPro.waitForCompilation();
 
@@ -150,7 +150,7 @@ public boolean reconfigure() {
 	 * @param bufferMap
 	 */
 	private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
-			ImmutableMap<Token, Buffer> bufferMap) {
+			ImmutableMap<Token, Buffer> bufferMap, int multiplier) {
 
 		TCPConnectionInfo headconInfo = conInfoMap.get(headToken);
 		assert headconInfo != null : "No head connection info exists in conInfoMap";
@@ -176,12 +176,12 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 			throw new IllegalArgumentException(
 					"No tail buffer in the passed bufferMap.");
 
+		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, GlobalConstants.outputCount,
+						+ tailToken.toString(), 0, skipCount,
 				GlobalConstants.outputCount);
 	}
-
 	/**
 	 * Start the execution of the StreamJit application.
 	 */
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index ad94ca25..fa118a8b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -102,7 +102,10 @@ public long getFixedOutputTime() throws InterruptedException {
 		Stopwatch stopwatch = Stopwatch.createStarted();
 		steadyLatch.await();
 		stopwatch.stop();
-		return stopwatch.elapsed(TimeUnit.MILLISECONDS);
+		long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+		long normalizedTime = (GlobalConstants.outputCount * time)
+				/ (totalCount - skipCount);
+		return normalizedTime;
 	}
 
 	public void reset() {
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index baca6047..4b53fbf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -12,6 +12,7 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -162,9 +163,11 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				}
 			}
 
+			int multiplier = config.getParameter("multiplier",
+					IntParameter.class).getValue();
 			drainer.setBlobGraph(app.blobGraph);
-			System.err.println("Reconfiguring...");
-			if (manager.reconfigure()) {
+			System.err.println("Reconfiguring...multiplier = " + multiplier);
+			if (manager.reconfigure(multiplier)) {
 				// TODO: need to check the manager's status before
 				// passing
 				// the time. Exceptions, final drain, etc may causes app

From d9534c367e45d1bb80247c73f46a256db51f9d9b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 02:30:01 +0800
Subject: [PATCH 162/881] GlobalConstants.TunerMode has been made final

---
 .../mit/streamjit/impl/distributed/DistributedAppRunner.java  | 2 --
 .../streamjit/impl/distributed/common/GlobalConstants.java    | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java b/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
index 852a87cf..5a42a848 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
@@ -64,8 +64,6 @@ public static void main(String[] args) throws InterruptedException {
 			}
 		}
 
-		GlobalConstants.tunerMode = 1;
-
 		Benchmark benchmark = bp.iterator().next();
 		// StreamCompiler compiler = new Compiler2StreamCompiler();
 		StreamCompiler compiler = new DistributedStreamCompiler(noOfNodes);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 9405c5ce..39bf5b1d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -70,7 +70,7 @@ private GlobalConstants() {
 	 * tuning on remote machines.
 	 * </ol>
 	 */
-	public static int tunerMode = 0;
+	public static final int tunerMode = 0;
 
 	/**
 	 * To turn on or turn off the drain data. If this is false, drain data will
@@ -121,7 +121,7 @@ private GlobalConstants() {
 	/**
 	 * Prints number of outputs generated.
 	 */
-	public static final boolean printOutputCount = true;
+	public static final boolean printOutputCount = false;
 
 	/**
 	 * Enables {@link DistributedStreamCompiler} to run on a single node. When

From 3d3780ee9b21d437512f293c993bd64be6a94db3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 11:45:08 +0800
Subject: [PATCH 163/881] Catches OutOfMemoryError when making blob

---
 .../distributed/node/CfgStringProcessorImpl.java   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 92082f8e..063e6255 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -4,6 +4,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -147,6 +150,17 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;
+				} catch (OutOfMemoryError er) {
+					MemoryMXBean memoryBean = ManagementFactory
+							.getMemoryMXBean();
+					System.out.println("******OutOfMemoryError******");
+					MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
+					int MEGABYTE = 1024 * 1024;
+					long maxMemory = heapUsage.getMax() / MEGABYTE;
+					long usedMemory = heapUsage.getUsed() / MEGABYTE;
+					System.out.println("Memory Use :" + usedMemory + "M/"
+							+ maxMemory + "M");
+					return null;
 				}
 			}
 			System.out.println("All blobs have been created");

From 0f14dd6ceccae0e272c6621a4839a72ce4578bfd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 11:51:26 +0800
Subject: [PATCH 164/881] A method that prints artition has been added.

---
 .../impl/distributed/StreamJitApp.java        | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 05ed5237..40640257 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -19,6 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
@@ -124,16 +125,11 @@ public boolean newPartitionMap(
 	 */
 	public void varifyConfiguration(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
-		for (int machine : partitionsMachineMap.keySet()) {
-			System.err.print("\nMachine - " + machine);
-			for (Set<Worker<?, ?>> blobworkers : partitionsMachineMap
-					.get(machine)) {
-				System.err.print("\n\tBlob worker set : ");
-				for (Worker<?, ?> w : blobworkers) {
-					System.err.print(Workers.getIdentifier(w) + " ");
-				}
-			}
+
+		if (!GlobalConstants.singleNodeOnline) {
+			printPartition(partitionsMachineMap);
 		}
+
 		List<Set<Worker<?, ?>>> partitionList = new ArrayList<>();
 		for (List<Set<Worker<?, ?>>> lst : partitionsMachineMap.values()) {
 			partitionList.addAll(lst);
@@ -143,24 +139,29 @@ public void varifyConfiguration(
 		try {
 			bg = new BlobGraph(partitionList);
 		} catch (StreamCompilationFailedException ex) {
-			System.err.print("Cycles found in the worker->blob assignment");
-			// for (int machine : partitionsMachineMap.keySet()) {
-			// System.err.print("\nMachine - " + machine);
-			// for (Set<Worker<?, ?>> blobworkers : partitionsMachineMap
-			// .get(machine)) {
-			// System.err.print("\n\tBlob worker set : ");
-			// for (Worker<?, ?> w : blobworkers) {
-			// System.err.print(Workers.getIdentifier(w) + " ");
-			// }
-			// }
-			// }
-			System.err.println();
+			System.err.println("Cycles found in the worker->blob assignment");
+			printPartition(partitionsMachineMap);
 			throw ex;
 		}
 		this.blobGraph = bg;
 		this.partitionsMachineMap = partitionsMachineMap;
 	}
 
+	private void printPartition(
+			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+		for (int machine : partitionsMachineMap.keySet()) {
+			System.err.print("\nMachine - " + machine);
+			for (Set<Worker<?, ?>> blobworkers : partitionsMachineMap
+					.get(machine)) {
+				System.err.print("\n\tBlob worker set : ");
+				for (Worker<?, ?> w : blobworkers) {
+					System.err.print(Workers.getIdentifier(w) + " ");
+				}
+			}
+		}
+		System.err.println();
+	}
+
 	/**
 	 * From aggregated drain data, get subset of it which is relevant to a
 	 * particular machine. Builds and returns machineID to DrainData map.

From 1af52befbfda56297cd41bcd341b5a22bb3a8298 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 11:53:45 +0800
Subject: [PATCH 165/881] spelling correction: varify -> verify

---
 src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java | 2 +-
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java  | 4 ++--
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index deff1201..29c24e4f 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -57,7 +57,7 @@ public boolean newConfiguration(Configuration config) {
 
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
 		try {
-			app.varifyConfiguration(partitionsMachineMap);
+			app.verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
 			return false;
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 40640257..77b5c7fe 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -103,7 +103,7 @@ public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
 	public boolean newPartitionMap(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
 		try {
-			varifyConfiguration(partitionsMachineMap);
+			verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
 			return false;
 		}
@@ -123,7 +123,7 @@ public boolean newPartitionMap(
 	 * @throws StreamCompilationFailedException
 	 *             if any cycles found among blobs.
 	 */
-	public void varifyConfiguration(
+	public void verifyConfiguration(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
 
 		if (!GlobalConstants.singleNodeOnline) {
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 841af384..7dcc7e5d 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -88,7 +88,7 @@ public boolean newConfiguration(Configuration config) {
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
 				config, app.source);
 		try {
-			app.varifyConfiguration(partitionsMachineMap);
+			app.verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
 			return false;
 		}

From a056fb3478c539a41d79b488bf88b8bf9c3acbda Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 12:10:07 +0800
Subject: [PATCH 166/881] Location of a debug print msg has been moved.

---
 .../impl/distributed/node/CfgStringProcessorImpl.java    | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 063e6255..4f98fd36 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -138,12 +138,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				// DEBUG MSG
-				System.out.println(String.format(
-						"A new blob with workers %s has been created.",
-						workIdentifiers.toString()));
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
-
 				try {
 					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
@@ -162,6 +157,10 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 							+ maxMemory + "M");
 					return null;
 				}
+				// DEBUG MSG
+				System.out.println(String.format(
+						"A new blob with workers %s has been created.",
+						workIdentifiers.toString()));
 			}
 			System.out.println("All blobs have been created");
 			return blobSet.build();

From 8cb6c63e050026a6e230ac17c16d634895fb331e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 12:14:23 +0800
Subject: [PATCH 167/881] Passes maximumThreadCount when making blobs

Previously just 1 was passed.
---
 .../impl/distributed/node/CfgStringProcessorImpl.java      | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 4f98fd36..ae97eecd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -136,12 +136,17 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
 
+			int maxThreadCount = Math.max(Runtime.getRuntime()
+					.availableProcessors() / 2, 1);
+			System.out.println(String.format("Maximum thread count = %d",
+					maxThreadCount));
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 				try {
 					BlobFactory bf = bs.getBlobFactory();
-					Blob b = bf.makeBlob(workerset, blobConfigs, 1, drainData);
+					Blob b = bf.makeBlob(workerset, blobConfigs,
+							maxThreadCount, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					return null;

From 9f7834561b377b718c438d5162a2b2407a214cc7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 16:02:18 +0800
Subject: [PATCH 168/881] GlobalConstants.maxNumCores has been added.

We shold pass this value to AllocationStrategy ALLOCATION_STRATEGY =
 new CompositionAllocationStrategy(GlobalConstants.maxNumCores); in
Compiler2.java. That change is not pushed.
---
 .../impl/distributed/common/GlobalConstants.java       | 10 ++++++++++
 .../impl/distributed/node/CfgStringProcessorImpl.java  |  7 ++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 39bf5b1d..674e7e8c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -129,6 +129,16 @@ private GlobalConstants() {
 	 */
 	public static final boolean singleNodeOnline = true;
 
+	/**
+	 * We can set this value at class loading time also as follows.
+	 * 
+	 * <code>maxThreadCount = Math.max(Runtime.getntime().availableProcessors() / 2,
+	 * 1);</code>
+	 * 
+	 * Lets hard code this for the moment.
+	 */
+	public static final int maxNumCores = 24;
+
 	static {
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index ae97eecd..6bad8086 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -136,19 +136,16 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
 
-			int maxThreadCount = Math.max(Runtime.getRuntime()
-					.availableProcessors() / 2, 1);
-			System.out.println(String.format("Maximum thread count = %d",
-					maxThreadCount));
 			for (BlobSpecifier bs : blobList) {
 				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 				try {
 					BlobFactory bf = bs.getBlobFactory();
 					Blob b = bf.makeBlob(workerset, blobConfigs,
-							maxThreadCount, drainData);
+							GlobalConstants.maxNumCores, drainData);
 					blobSet.add(b);
 				} catch (Exception ex) {
+					ex.printStackTrace();
 					return null;
 				} catch (OutOfMemoryError er) {
 					MemoryMXBean memoryBean = ManagementFactory

From 609b24d49298ed31f81e25040202111e0b4f21ab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 15 Mar 2014 13:47:29 +0800
Subject: [PATCH 169/881] Handles run time errors and exceptions.

Blob may fail to run and throw either error or exception. These events are
captured and reported to Controller.
---
 .../impl/distributed/StreamJitAppManager.java | 10 +++-
 .../distributed/node/BlobsManagerImpl.java    | 55 +++++++++++++++++--
 2 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index fa59d09d..deb6a7ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -254,7 +254,10 @@ else if (GlobalConstants.useDrainData)
 	}
 
 	public long getFixedOutputTime() throws InterruptedException {
-		return tailChannel.getFixedOutputTime();
+		long time = tailChannel.getFixedOutputTime();
+		if (apStsPro.error)
+			return -1l;
+		return time;
 	}
 
 	public void setDrainer(AbstractDrainer drainer) {
@@ -404,6 +407,8 @@ private class AppStatusProcessorImpl implements AppStatusProcessor {
 
 		private boolean compilationError;
 
+		private volatile boolean error;
+
 		private final int noOfnodes;
 
 		private AppStatusProcessorImpl(int noOfnodes) {
@@ -420,6 +425,8 @@ public void processSTOPPED() {
 
 		@Override
 		public void processERROR() {
+			this.error = true;
+			tailChannel.reset();
 		}
 
 		@Override
@@ -445,6 +452,7 @@ public void processCOMPILATION_ERROR() {
 		private void reset() {
 			compileLatch = new CountDownLatch(noOfnodes);
 			this.compilationError = false;
+			this.error = false;
 		}
 
 		private boolean waitForCompilation() {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f72bb157..58c85ffa 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -239,7 +239,7 @@ private class BlobExecuter {
 		private final Token blobID;
 
 		private final Blob blob;
-		private Set<BlobThread> blobThreads;
+		private Set<BlobThread2> blobThreads;
 
 		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
 		private final ImmutableMap<Token, BoundaryOutputChannel> outputChannels;
@@ -267,7 +267,7 @@ private BlobExecuter(Blob blob,
 					sb.append(Workers.getIdentifier(w));
 					sb.append(",");
 				}
-				blobThreads.add(new BlobThread(blob.getCoreCode(i), sb
+				blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, sb
 						.toString()));
 			}
 
@@ -346,8 +346,12 @@ private void doDrain(boolean reqDrainData) {
 		}
 
 		private void drained() {
-			drainState = 3;
-			for (BlobThread bt : blobThreads) {
+			if (drainState < 3)
+				drainState = 3;
+			else
+				return;
+
+			for (BlobThread2 bt : blobThreads) {
 				bt.requestStop();
 			}
 
@@ -363,6 +367,9 @@ private void drained() {
 				}
 			}
 
+			if (drainState > 3)
+				return;
+
 			drainState = 4;
 			SNMessageElement drained = new SNDrainElement.Drained(blobID);
 			try {
@@ -609,6 +616,46 @@ public void processSTOP() {
 		}
 	}
 
+	private final class BlobThread2 extends Thread {
+		private volatile boolean stopping = false;
+		private final Runnable coreCode;
+		private final BlobExecuter be;
+
+		private BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
+			super(name);
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		private BlobThread2(Runnable coreCode, BlobExecuter be) {
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		@Override
+		public void run() {
+			try {
+				while (!stopping)
+					coreCode.run();
+			} catch (Error | Exception e) {
+				e.printStackTrace();
+				if (be.drainState == 1 || be.drainState == 2)
+					be.drained();
+				else if (be.drainState == 0) {
+					try {
+						streamNode.controllerConnection
+								.writeObject(AppStatus.ERROR);
+					} catch (IOException e1) {
+						e1.printStackTrace();
+					}
+				}
+			}
+		}
+		public void requestStop() {
+			stopping = true;
+		}
+	}
+
 	private static int count = 0;
 
 	private class MonitorBuffers extends Thread {

From 972cda3981ec820be4213f7b5a143b19faadb516 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 15 Mar 2014 22:24:35 +0800
Subject: [PATCH 170/881] Sometimes Tcp connection passes null.

It has been handled here.
---
 .../mit/streamjit/impl/distributed/node/TCPInputChannel.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index b6c024d2..67084278 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -145,6 +145,8 @@ public void receiveData() {
 		int bufFullCount = 0;
 		try {
 			Object obj = tcpConnection.readObject();
+			if(obj == null)	// [2014-03-15] Sometimes null is received.
+				return;
 			count++;
 
 			if (debugLevel == 3) {

From 832135301641e28514ce314c7fee590e986ee072 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 16 Mar 2014 01:18:43 +0800
Subject: [PATCH 171/881] Meassures and prints draining time

Just for observation purpose.
---
 .../impl/distributed/StreamJitAppManager.java    | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index deb6a7ac..3a2288f6 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -5,7 +5,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.api.CompiledStream;
@@ -209,7 +212,13 @@ public boolean isRunning() {
 		return isRunning;
 	}
 
+	/**
+	 * [2014-03-15] Just to measure the draining time
+	 */
+	AtomicReference<Stopwatch> stopwatchRef = new AtomicReference<>();
+
 	public void drainingStarted(boolean isFinal) {
+		stopwatchRef.set(Stopwatch.createStarted());
 		if (headChannel != null) {
 			headChannel.stop(isFinal);
 			try {
@@ -251,6 +260,13 @@ else if (GlobalConstants.useDrainData)
 			controller.closeAll();
 		}
 		isRunning = false;
+
+		Stopwatch sw = stopwatchRef.get();
+		if (sw != null) {
+			sw.stop();
+			long time = sw.elapsed(TimeUnit.MILLISECONDS);
+			System.out.println("Draining time is " + time + " milli seconds");
+		}
 	}
 
 	public long getFixedOutputTime() throws InterruptedException {

From 60cc8eb4cd0d5e83e91eaafdc66caceb5e283b00 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 11:45:08 +0800
Subject: [PATCH 172/881] Catches OutOfMemoryError when making blob

---
 .../impl/distributed/node/CfgStringProcessorImpl.java | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 6bad8086..bcf9887a 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -158,6 +158,17 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					System.out.println("Memory Use :" + usedMemory + "M/"
 							+ maxMemory + "M");
 					return null;
+				} catch (OutOfMemoryError er) {
+					MemoryMXBean memoryBean = ManagementFactory
+							.getMemoryMXBean();
+					System.out.println("******OutOfMemoryError******");
+					MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
+					int MEGABYTE = 1024 * 1024;
+					long maxMemory = heapUsage.getMax() / MEGABYTE;
+					long usedMemory = heapUsage.getUsed() / MEGABYTE;
+					System.out.println("Memory Use :" + usedMemory + "M/"
+							+ maxMemory + "M");
+					return null;
 				}
 				// DEBUG MSG
 				System.out.println(String.format(

From f871f2e95021f2021a89f45c15767e52bbb3c4fa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 14 Mar 2014 16:02:18 +0800
Subject: [PATCH 173/881] GlobalConstants.maxNumCores has been added.

We shold pass this value to AllocationStrategy ALLOCATION_STRATEGY =
 new CompositionAllocationStrategy(GlobalConstants.maxNumCores); in
Compiler2.java. That change is not pushed.
---
 .../impl/distributed/node/CfgStringProcessorImpl.java | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index bcf9887a..6bad8086 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -158,17 +158,6 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					System.out.println("Memory Use :" + usedMemory + "M/"
 							+ maxMemory + "M");
 					return null;
-				} catch (OutOfMemoryError er) {
-					MemoryMXBean memoryBean = ManagementFactory
-							.getMemoryMXBean();
-					System.out.println("******OutOfMemoryError******");
-					MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
-					int MEGABYTE = 1024 * 1024;
-					long maxMemory = heapUsage.getMax() / MEGABYTE;
-					long usedMemory = heapUsage.getUsed() / MEGABYTE;
-					System.out.println("Memory Use :" + usedMemory + "M/"
-							+ maxMemory + "M");
-					return null;
 				}
 				// DEBUG MSG
 				System.out.println(String.format(

From 715778192eebb2ab07bd46f47954e96902ba7830 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 16 Mar 2014 13:55:32 +0800
Subject: [PATCH 174/881] dumps the drain data into a file.

---
 .../mit/streamjit/impl/common/AbstractDrainer.java   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 35abf078..beadba0c 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -2,8 +2,10 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -269,9 +271,17 @@ public final DrainData getDrainData() {
 			drainDataStatistics.get(t).add(drainData.getData().get(t).size());
 		}
 
+		// dumps the drain data into a file.
+		// try {
+		// FileOutputStream fout = new FileOutputStream("DrainData");
+		// ObjectOutputStream oos = new ObjectOutputStream(fout);
+		// oos.writeObject(drainData);
+		// oos.close();
+		// } catch (Exception ex) {
+		// ex.printStackTrace();
+		// }
 		return drainData;
 	}
-
 	/**
 	 * logs the size of the drain data on each channel for every draining and
 	 * writes down the statistics into a file.

From 73810a7b59a3b04699e56f6df589dc532e20f458 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 16 Mar 2014 20:54:19 +0800
Subject: [PATCH 175/881] Merge() returns this DrainData if other DrainData

is null.
---
 src/edu/mit/streamjit/impl/blob/DrainData.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/blob/DrainData.java b/src/edu/mit/streamjit/impl/blob/DrainData.java
index 64869222..9958d4c7 100644
--- a/src/edu/mit/streamjit/impl/blob/DrainData.java
+++ b/src/edu/mit/streamjit/impl/blob/DrainData.java
@@ -65,6 +65,8 @@ public Object getWorkerState(int workerId, String fieldName) {
 	 * @return a merged DrainData
 	 */
 	public DrainData merge(DrainData other) {
+		if (other == null)
+			return this;
 		ImmutableMap.Builder<Token, ImmutableList<Object>> dataBuilder = ImmutableMap.builder();
 		for (Token t : Sets.union(data.keySet(), other.data.keySet())) {
 			ImmutableList<Object> us = getData(t) != null ? getData(t) : ImmutableList.of();

From aa18abb0ffbe58a2c8ca08497e4c0f7edc68efcc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 16 Mar 2014 21:12:39 +0800
Subject: [PATCH 176/881] StreamNodes send Error msg if a thread crashed.

When a blob thread crashed during the execution, it informs it to Controller
so that controller can discard the current tuning trial and move forward for
the next tuning.
---
 .../impl/distributed/StreamJitAppManager.java |  7 +++-
 .../impl/distributed/TailChannel.java         | 14 +++++++
 .../distributed/node/BlobsManagerImpl.java    | 41 ++++++++++++-------
 3 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 3a2288f6..536a09b9 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -271,8 +271,9 @@ else if (GlobalConstants.useDrainData)
 
 	public long getFixedOutputTime() throws InterruptedException {
 		long time = tailChannel.getFixedOutputTime();
-		if (apStsPro.error)
+		if (apStsPro.error) {
 			return -1l;
+		}
 		return time;
 	}
 
@@ -442,7 +443,9 @@ public void processSTOPPED() {
 		@Override
 		public void processERROR() {
 			this.error = true;
-			tailChannel.reset();
+			// This will release the OpenTuner thread which is waiting for fixed
+			// output.
+			tailChannel.releaseAll();
 		}
 
 		@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index fa118a8b..c7a4f6df 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -9,6 +9,7 @@
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
@@ -117,6 +118,19 @@ public void reset() {
 		skipLatchUp = true;
 	}
 
+	/**
+	 * This method is needed apart from {@link #reset()} because if a thread
+	 * calls (From {@link AppStatusProcessor#processERROR()} reset() method
+	 * instead of this method, the threads which are waiting on latches at
+	 * {@link #getFixedOutputTime()} will not be released properly because the
+	 * thread thread which is waiting on skipLatch will wait on steady latch.
+	 */
+	public void releaseAll() {
+		steadyLatch.countDown();
+		skipLatch.countDown();
+		count = 0;
+	}
+
 	private class performanceLogger extends Thread {
 
 		private AtomicBoolean stopFlag;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 58c85ffa..6c5159ff 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -235,6 +235,7 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 
 	private class BlobExecuter {
 
+		private AtomicBoolean crashed;
 		private volatile int drainState;
 		private final Token blobID;
 
@@ -252,6 +253,7 @@ private class BlobExecuter {
 		private BlobExecuter(Blob blob,
 				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
 				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
+			this.crashed = new AtomicBoolean(false);
 			this.blob = blob;
 			this.blobThreads = new HashSet<>();
 			assert blob.getInputs().containsAll(inputChannels.keySet());
@@ -316,6 +318,7 @@ private void stop() {
 		}
 
 		private void doDrain(boolean reqDrainData) {
+			// System.out.println("Blob " + blobID + "is doDrain");
 			this.reqDrainData = reqDrainData;
 			drainState = 1;
 
@@ -339,13 +342,16 @@ private void doDrain(boolean reqDrainData) {
 					e.printStackTrace();
 				}
 			}
-
+			// System.out.println("Blob " + blobID + "Input thread's joined");
 			DrainCallback dcb = new DrainCallback(this);
 			drainState = 2;
 			this.blob.drain(dcb);
+			// System.out.println("Blob " + blobID +
+			// "this.blob.drain(dcb); passed");
 		}
 
 		private void drained() {
+			// System.out.println("Blob " + blobID + "drained at beg");
 			if (drainState < 3)
 				drainState = 3;
 			else
@@ -377,17 +383,18 @@ private void drained() {
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
-			// System.out.println("Blob " + blobID + "is drained");
+			// System.out.println("Blob " + blobID + "is drained at mid");
 
 			if (GlobalConstants.useDrainData && this.reqDrainData) {
 				// System.out.println("**********************************");
 				DrainData dd = blob.getDrainData();
 				drainState = 5;
 
-				for (Token t : dd.getData().keySet()) {
-					System.out.println("From Blob: " + t.toString() + " - "
-							+ dd.getData().get(t).size());
-				}
+				if (dd != null)
+					for (Token t : dd.getData().keySet()) {
+						System.out.println("From Blob: " + t.toString() + " - "
+								+ dd.getData().get(t).size());
+					}
 
 				ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 				ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
@@ -638,15 +645,19 @@ public void run() {
 				while (!stopping)
 					coreCode.run();
 			} catch (Error | Exception e) {
-				e.printStackTrace();
-				if (be.drainState == 1 || be.drainState == 2)
-					be.drained();
-				else if (be.drainState == 0) {
-					try {
-						streamNode.controllerConnection
-								.writeObject(AppStatus.ERROR);
-					} catch (IOException e1) {
-						e1.printStackTrace();
+				System.out.println(Thread.currentThread().getName()
+						+ " crached...");
+				if (be.crashed.compareAndSet(false, true)) {
+					e.printStackTrace();
+					if (be.drainState == 1 || be.drainState == 2)
+						be.drained();
+					else if (be.drainState == 0) {
+						try {
+							streamNode.controllerConnection
+									.writeObject(AppStatus.ERROR);
+						} catch (IOException e1) {
+							e1.printStackTrace();
+						}
 					}
 				}
 			}

From 81d6c5015e85ad466273db223053a939ab88a432 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Mar 2014 00:56:16 +0800
Subject: [PATCH 177/881] headThread.join() is moved from startDraining...

method to drainingFinished method.
---
 .../impl/distributed/StreamJitAppManager.java | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 536a09b9..d1ed76cf 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -221,11 +221,13 @@ public void drainingStarted(boolean isFinal) {
 		stopwatchRef.set(Stopwatch.createStarted());
 		if (headChannel != null) {
 			headChannel.stop(isFinal);
-			try {
-				headThread.join();
-			} catch (InterruptedException e) {
-				e.printStackTrace();
-			}
+			// [2014-03-16] Moved to drainingFinished. In any case if headThread
+			// blocked at tcp write, draining will also blocked.
+			// try {
+			// headThread.join();
+			// } catch (InterruptedException e) {
+			// e.printStackTrace();
+			// }
 		}
 	}
 
@@ -241,6 +243,15 @@ public void drain(Token blobID, boolean isFinal) {
 
 	public void drainingFinished(boolean isFinal) {
 		System.out.println("App Manager : Draining Finished...");
+
+		if (headChannel != null) {
+			try {
+				headThread.join();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
 		if (tailChannel != null) {
 			if (isFinal)
 				tailChannel.stop(1);

From 4f6fd126753ad453a096efd0abfbcf17bd9783e5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Mar 2014 01:45:54 +0800
Subject: [PATCH 178/881] Debug print msgs commented

---
 .../impl/distributed/StreamJitApp.java        |  2 +-
 .../distributed/node/BlobsManagerImpl.java    | 31 ++++++++++---------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 77b5c7fe..ea3ebf60 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -127,7 +127,7 @@ public void verifyConfiguration(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
 
 		if (!GlobalConstants.singleNodeOnline) {
-			printPartition(partitionsMachineMap);
+			// printPartition(partitionsMachineMap);
 		}
 
 		List<Set<Worker<?, ?>>> partitionList = new ArrayList<>();
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 6c5159ff..33ae2eb1 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.common.BlobThread;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
@@ -95,7 +94,7 @@ public void start() {
 			be.start();
 
 		if (monBufs == null) {
-			System.out.println("Creating new MonitorBuffers");
+			// System.out.println("Creating new MonitorBuffers");
 			monBufs = new MonitorBuffers();
 			monBufs.start();
 		} else
@@ -172,7 +171,7 @@ private void addBuffer(Token t, int minSize,
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder) {
 		// TODO: Just to increase the performance. Change it later
 		int bufSize = Math.max(1000, minSize);
-		System.out.println("Buffer size of " + t.toString() + " is " + bufSize);
+		// System.out.println("Buffer size of " + t.toString() + " is " + bufSize);
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
@@ -390,11 +389,12 @@ private void drained() {
 				DrainData dd = blob.getDrainData();
 				drainState = 5;
 
-				if (dd != null)
-					for (Token t : dd.getData().keySet()) {
-						System.out.println("From Blob: " + t.toString() + " - "
-								+ dd.getData().get(t).size());
-					}
+				if (dd != null) {
+					// for (Token t : dd.getData().keySet()) {
+					// System.out.println("From Blob: " + t.toString() + " - "
+					// + dd.getData().get(t).size());
+					// }
+				}
 
 				ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 				ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
@@ -404,9 +404,9 @@ private void drained() {
 						BoundaryChannel chanl = inputChannels.get(t);
 						ImmutableList<Object> draindata = chanl
 								.getUnprocessedData();
-						System.out.println(String.format(
-								"No of unprocessed data of %s is %d",
-								chanl.name(), draindata.size()));
+						// System.out.println(String.format(
+						// "No of unprocessed data of %s is %d",
+						// chanl.name(), draindata.size()));
 						inputDataBuilder.put(t, draindata);
 					}
 
@@ -427,9 +427,9 @@ private void drained() {
 						BoundaryChannel chanl = outputChannels.get(t);
 						ImmutableList<Object> draindata = chanl
 								.getUnprocessedData();
-						System.out.println(String.format(
-								"No of unprocessed data of %s is %d",
-								chanl.name(), draindata.size()));
+						// System.out.println(String.format(
+						// "No of unprocessed data of %s is %d",
+						// chanl.name(), draindata.size()));
 						outputDataBuilder.put(t, draindata);
 					}
 				}
@@ -674,6 +674,7 @@ private class MonitorBuffers extends Thread {
 		private final AtomicBoolean stopFlag;
 		int sleepTime = 25000;
 		MonitorBuffers() {
+			super("MonitorBuffers");
 			stopFlag = new AtomicBoolean(false);
 			id = count++;
 		}
@@ -723,7 +724,7 @@ public void run() {
 		}
 
 		public void stopMonitoring() {
-			System.out.println("MonitorBuffers: Stop monitoring");
+			// System.out.println("MonitorBuffers: Stop monitoring");
 			stopFlag.set(true);
 			this.interrupt();
 		}

From 5246a847937081d6a4b8ae2a0c2662a0b96c3a67 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Mar 2014 23:56:29 +0800
Subject: [PATCH 179/881] ObjectOutputStream.reset() count has been reduced.

It was 50 for StreamNode conncetion, controller connection. But when
drain data is sent it often triggers out of memory error. Now this reset
count is reduced to 5.
---
 .../impl/distributed/common/SynchronizedTCPConnection.java     | 2 +-
 .../mit/streamjit/impl/distributed/common/TCPConnection.java   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/SynchronizedTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/SynchronizedTCPConnection.java
index 0dd84df6..5e25aa7d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SynchronizedTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SynchronizedTCPConnection.java
@@ -21,7 +21,7 @@ public class SynchronizedTCPConnection extends TCPConnection {
 	 * @param socket
 	 */
 	public SynchronizedTCPConnection(Socket socket) {
-		super(socket, 50);
+		super(socket, 5);
 	}
 
 	@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 52bf30ca..40bd79db 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -64,8 +64,9 @@ public void writeObject(Object obj) throws IOException {
 			try {
 				ooStream.writeObject(obj);
 
+				n++;
 				// TODO: Any way to improve the performance?
-				if (n++ > resetCount) {
+				if (n > resetCount) {
 					n = 0;
 					ooStream.reset();
 				}

From 815f641132bca61bf3d6f760cd1858d64125d686 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 18 Mar 2014 00:15:46 +0800
Subject: [PATCH 180/881] Made final

---
 .../distributed/runtimer/BlockingCommunicationManager.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
index 152b2fe8..e24fe5db 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
@@ -181,8 +181,8 @@ public InetAddress getAddress() {
 	 * 
 	 */
 	private static class SNAgentRunner extends Thread {
-		StreamNodeAgent SNAgent;
-		Connection connection;
+		final StreamNodeAgent SNAgent;
+		final Connection connection;
 
 		private SNAgentRunner(StreamNodeAgent SNAgent, Connection connection) {
 			super(String.format("SNAgentRunner - %d", SNAgent.getNodeID()));

From 8c10bfb1431fbce3853f9cf5f9715553eded547a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 18 Mar 2014 01:09:19 +0800
Subject: [PATCH 181/881] Refactored.

Large method drained() has been splitted in to 2.
---
 .../distributed/node/BlobsManagerImpl.java    | 110 +++++++++---------
 1 file changed, 57 insertions(+), 53 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 33ae2eb1..3d1a1953 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -31,6 +31,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
@@ -171,7 +172,8 @@ private void addBuffer(Token t, int minSize,
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder) {
 		// TODO: Just to increase the performance. Change it later
 		int bufSize = Math.max(1000, minSize);
-		// System.out.println("Buffer size of " + t.toString() + " is " + bufSize);
+		// System.out.println("Buffer size of " + t.toString() + " is " +
+		// bufSize);
 		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
 	}
 
@@ -385,57 +387,7 @@ private void drained() {
 			// System.out.println("Blob " + blobID + "is drained at mid");
 
 			if (GlobalConstants.useDrainData && this.reqDrainData) {
-				// System.out.println("**********************************");
-				DrainData dd = blob.getDrainData();
-				drainState = 5;
-
-				if (dd != null) {
-					// for (Token t : dd.getData().keySet()) {
-					// System.out.println("From Blob: " + t.toString() + " - "
-					// + dd.getData().get(t).size());
-					// }
-				}
-
-				ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
-				ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
-
-				for (Token t : blob.getInputs()) {
-					if (inputChannels.containsKey(t)) {
-						BoundaryChannel chanl = inputChannels.get(t);
-						ImmutableList<Object> draindata = chanl
-								.getUnprocessedData();
-						// System.out.println(String.format(
-						// "No of unprocessed data of %s is %d",
-						// chanl.name(), draindata.size()));
-						inputDataBuilder.put(t, draindata);
-					}
-
-					// TODO: Unnecessary data copy. Optimise this.
-					else {
-						Buffer buf = bufferMap.get(t);
-						Object[] bufArray = new Object[buf.size()];
-						buf.readAll(bufArray);
-						assert buf.size() == 0 : String.format(
-								"buffer size is %d. But 0 is expected",
-								buf.size());
-						inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
-					}
-				}
-
-				for (Token t : blob.getOutputs()) {
-					if (outputChannels.containsKey(t)) {
-						BoundaryChannel chanl = outputChannels.get(t);
-						ImmutableList<Object> draindata = chanl
-								.getUnprocessedData();
-						// System.out.println(String.format(
-						// "No of unprocessed data of %s is %d",
-						// chanl.name(), draindata.size()));
-						outputDataBuilder.put(t, draindata);
-					}
-				}
-
-				SNMessageElement me = new SNDrainElement.DrainedData(blobID,
-						dd, inputDataBuilder.build(), outputDataBuilder.build());
+				SNMessageElement me = getDrainData();
 				try {
 					streamNode.controllerConnection.writeObject(me);
 					// System.out.println(blobID + " DrainData has been sent");
@@ -444,7 +396,6 @@ private void drained() {
 				} catch (IOException e) {
 					e.printStackTrace();
 				}
-
 				// System.out.println("**********************************");
 			}
 
@@ -462,6 +413,59 @@ private void drained() {
 			// printDrainedStatus();
 		}
 
+		private DrainedData getDrainData() {
+			// System.out.println("**********************************");
+			DrainData dd = blob.getDrainData();
+			drainState = 5;
+
+			if (dd != null) {
+				// for (Token t : dd.getData().keySet()) {
+				// System.out.println("From Blob: " + t.toString() + " - "
+				// + dd.getData().get(t).size());
+				// }
+			}
+
+			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
+			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
+
+			for (Token t : blob.getInputs()) {
+				if (inputChannels.containsKey(t)) {
+					BoundaryChannel chanl = inputChannels.get(t);
+					ImmutableList<Object> draindata = chanl
+							.getUnprocessedData();
+					// System.out.println(String.format(
+					// "No of unprocessed data of %s is %d",
+					// chanl.name(), draindata.size()));
+					inputDataBuilder.put(t, draindata);
+				}
+
+				// TODO: Unnecessary data copy. Optimise this.
+				else {
+					Buffer buf = bufferMap.get(t);
+					Object[] bufArray = new Object[buf.size()];
+					buf.readAll(bufArray);
+					assert buf.size() == 0 : String.format(
+							"buffer size is %d. But 0 is expected", buf.size());
+					inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
+				}
+			}
+
+			for (Token t : blob.getOutputs()) {
+				if (outputChannels.containsKey(t)) {
+					BoundaryChannel chanl = outputChannels.get(t);
+					ImmutableList<Object> draindata = chanl
+							.getUnprocessedData();
+					// System.out.println(String.format(
+					// "No of unprocessed data of %s is %d",
+					// chanl.name(), draindata.size()));
+					outputDataBuilder.put(t, draindata);
+				}
+			}
+
+			return new SNDrainElement.DrainedData(blobID, dd,
+					inputDataBuilder.build(), outputDataBuilder.build());
+		}
+
 		public Token getBlobID() {
 			return Utils.getBlobID(blob);
 		}

From 2bc242f80e706f8354d9dd6bdf4410b13a5331ee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 18 Mar 2014 01:15:46 +0800
Subject: [PATCH 182/881] Sends empty draindata if a blobthread is crashed

---
 .../distributed/node/BlobsManagerImpl.java    | 22 ++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 3d1a1953..fc5000cf 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -11,6 +11,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Sets;
 
 import edu.mit.streamjit.api.Worker;
@@ -387,7 +388,12 @@ private void drained() {
 			// System.out.println("Blob " + blobID + "is drained at mid");
 
 			if (GlobalConstants.useDrainData && this.reqDrainData) {
-				SNMessageElement me = getDrainData();
+				SNMessageElement me;
+				if (crashed.get())
+					me = getEmptyDrainData();
+				else
+					me = getDrainData();
+
 				try {
 					streamNode.controllerConnection.writeObject(me);
 					// System.out.println(blobID + " DrainData has been sent");
@@ -466,6 +472,20 @@ private DrainedData getDrainData() {
 					inputDataBuilder.build(), outputDataBuilder.build());
 		}
 
+		private DrainedData getEmptyDrainData() {
+			drainState = 5;
+			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
+			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
+			ImmutableMap.Builder<Token, ImmutableList<Object>> dataBuilder = ImmutableMap
+					.builder();
+			ImmutableTable.Builder<Integer, String, Object> stateBuilder = ImmutableTable
+					.builder();
+			DrainData dd = new DrainData(dataBuilder.build(),
+					stateBuilder.build());
+			return new SNDrainElement.DrainedData(blobID, dd,
+					inputDataBuilder.build(), outputDataBuilder.build());
+		}
+
 		public Token getBlobID() {
 			return Utils.getBlobID(blob);
 		}

From 09a69a8b3d365c79ce70d0e26d8a3593ef973751 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 18 Mar 2014 02:49:57 +0800
Subject: [PATCH 183/881] Meassures and prints draining time of a blob

---
 .../impl/distributed/node/BlobsManagerImpl.java          | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index fc5000cf..4964b157 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -6,8 +6,10 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -494,13 +496,20 @@ public Token getBlobID() {
 	private static class DrainCallback implements Runnable {
 
 		private final BlobExecuter blobExec;
+		// TODO: [2014-03-17] Just to added for checking the drain time. Remove
+		// it later.
+		private final Stopwatch sw;
 
 		DrainCallback(BlobExecuter be) {
 			this.blobExec = be;
+			sw = Stopwatch.createStarted();
 		}
 
 		@Override
 		public void run() {
+			sw.stop();
+			System.out.println("Time taken to drain " + blobExec.blobID
+					+ " is " + sw.elapsed(TimeUnit.MILLISECONDS) + " ms");
 			blobExec.drained();
 		}
 	}

From 4eea9b8b7b1c1a5b14d5e2e94b4f0f6bac89d1f2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 19 Mar 2014 12:33:45 +0800
Subject: [PATCH 184/881] blobExecuters: changed from set -> map

blobExcuters keep references to blob and blobs are often huge objects that
sometimes causes out of memory error too. Having BlobExcuters ( which has blobs
inside) as a key to the hashmap (this hashmap is one which backs hashset
implementation) causes dereferencing and release the memory hard.
---
 .../distributed/node/BlobsManagerImpl.java    | 47 ++++++++++---------
 1 file changed, 26 insertions(+), 21 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 4964b157..f1f87ec9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -48,7 +48,7 @@
  */
 public class BlobsManagerImpl implements BlobsManager {
 
-	private Set<BlobExecuter> blobExecuters;
+	private Map<Token, BlobExecuter> blobExecuters;
 	private final StreamNode streamNode;
 	private final TCPConnectionProvider conProvider;
 	private Map<Token, TCPConnectionInfo> conInfoMap;
@@ -78,14 +78,15 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		}
 
 		Set<Token> locaTokens = getLocalTokens(blobSet);
-		blobExecuters = new HashSet<>();
+		blobExecuters = new HashMap<>();
 		for (Blob b : blobSet) {
+			Token t = Utils.getBlobID(b);
 			ImmutableMap<Token, BoundaryInputChannel> inputChannels = createInputChannels(
 					Sets.difference(b.getInputs(), locaTokens), bufferMap);
 			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = createOutputChannels(
 					Sets.difference(b.getOutputs(), locaTokens), bufferMap);
-			blobExecuters
-					.add(new BlobExecuter(b, inputChannels, outputChannels));
+			blobExecuters.put(t, new BlobExecuter(t, b, inputChannels,
+					outputChannels));
 		}
 	}
 
@@ -94,16 +95,16 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 	 * manage all CPU and I/O threads those are related to the {@link Blob}s.
 	 */
 	public void start() {
-		for (BlobExecuter be : blobExecuters)
+		for (BlobExecuter be : blobExecuters.values())
 			be.start();
 
-		if (monBufs == null) {
-			// System.out.println("Creating new MonitorBuffers");
-			monBufs = new MonitorBuffers();
-			monBufs.start();
-		} else
-			System.err
-					.println("Mon buffer is not null. Check the logic for bug");
+		// if (monBufs == null) {
+		// // System.out.println("Creating new MonitorBuffers");
+		// monBufs = new MonitorBuffers();
+		// monBufs.start();
+		// } else
+		// System.err
+		// .println("Mon buffer is not null. Check the logic for bug");
 	}
 
 	/**
@@ -111,7 +112,7 @@ public void start() {
 	 * stopped.
 	 */
 	public void stop() {
-		for (BlobExecuter be : blobExecuters)
+		for (BlobExecuter be : blobExecuters.values())
 			be.stop();
 
 		if (monBufs != null)
@@ -243,7 +244,7 @@ private class BlobExecuter {
 		private volatile int drainState;
 		private final Token blobID;
 
-		private final Blob blob;
+		private Blob blob;
 		private Set<BlobThread2> blobThreads;
 
 		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
@@ -254,7 +255,7 @@ private class BlobExecuter {
 
 		private boolean reqDrainData;
 
-		private BlobExecuter(Blob blob,
+		private BlobExecuter(Token t, Blob blob,
 				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
 				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
 			this.crashed = new AtomicBoolean(false);
@@ -269,16 +270,19 @@ private BlobExecuter(Blob blob,
 
 			for (int i = 0; i < blob.getCoreCount(); i++) {
 				StringBuilder sb = new StringBuilder("Workers-");
+				int limit = 0;
 				for (Worker<?, ?> w : blob.getWorkers()) {
 					sb.append(Workers.getIdentifier(w));
 					sb.append(",");
+					if (++limit > 5)
+						break;
 				}
 				blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, sb
 						.toString()));
 			}
 
 			drainState = 0;
-			this.blobID = Utils.getBlobID(blob);
+			this.blobID = t;
 		}
 
 		private void start() {
@@ -407,8 +411,9 @@ private void drained() {
 				// System.out.println("**********************************");
 			}
 
+			this.blob = null;
 			boolean isLastBlob = true;
-			for (BlobExecuter be : blobExecuters) {
+			for (BlobExecuter be : blobExecuters.values()) {
 				if (be.drainState < 4) {
 					isLastBlob = false;
 					break;
@@ -489,7 +494,7 @@ private DrainedData getEmptyDrainData() {
 		}
 
 		public Token getBlobID() {
-			return Utils.getBlobID(blob);
+			return blobID;
 		}
 	}
 
@@ -518,7 +523,7 @@ public void run() {
 	 * Drain the blob identified by the token.
 	 */
 	public void drain(Token blobID, boolean reqDrainData) {
-		for (BlobExecuter be : blobExecuters) {
+		for (BlobExecuter be : blobExecuters.values()) {
 			if (be.getBlobID().equals(blobID)) {
 				be.doDrain(reqDrainData);
 				return;
@@ -533,7 +538,7 @@ public void drain(Token blobID, boolean reqDrainData) {
 	 */
 	private synchronized void printDrainedStatus() {
 		System.out.println("****************************************");
-		for (BlobExecuter be : blobExecuters) {
+		for (BlobExecuter be : blobExecuters.values()) {
 			switch (be.drainState) {
 				case 0 :
 					System.out.println(String.format("%s - No Drain Called",
@@ -679,7 +684,7 @@ public void run() {
 					coreCode.run();
 			} catch (Error | Exception e) {
 				System.out.println(Thread.currentThread().getName()
-						+ " crached...");
+						+ " crashed...");
 				if (be.crashed.compareAndSet(false, true)) {
 					e.printStackTrace();
 					if (be.drainState == 1 || be.drainState == 2)

From f2f781b71e494398fe680f6c189e137192096615 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 19 Mar 2014 12:51:20 +0800
Subject: [PATCH 185/881] Stop printing blob workers in Single node case

---
 .../impl/distributed/node/CfgStringProcessorImpl.java      | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 6bad8086..827f13ea 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -160,9 +160,10 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					return null;
 				}
 				// DEBUG MSG
-				System.out.println(String.format(
-						"A new blob with workers %s has been created.",
-						workIdentifiers.toString()));
+				if (!GlobalConstants.singleNodeOnline)
+					System.out.println(String.format(
+							"A new blob with workers %s has been created.",
+							workIdentifiers.toString()));
 			}
 			System.out.println("All blobs have been created");
 			return blobSet.build();

From 533a1948b74dad091e815649e74b0acaca132875 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 20 Mar 2014 09:03:19 +0800
Subject: [PATCH 186/881] Checks for null before accessing

Now we intentionally sets blob=null to free the memory. So other threads
should check for nullness before accessing it.
---
 .../impl/distributed/node/BlobsManagerImpl.java        | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f1f87ec9..07308b60 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -351,9 +351,11 @@ private void doDrain(boolean reqDrainData) {
 				}
 			}
 			// System.out.println("Blob " + blobID + "Input thread's joined");
-			DrainCallback dcb = new DrainCallback(this);
-			drainState = 2;
-			this.blob.drain(dcb);
+			if (this.blob != null) {
+				DrainCallback dcb = new DrainCallback(this);
+				drainState = 2;
+				this.blob.drain(dcb);
+			}
 			// System.out.println("Blob " + blobID +
 			// "this.blob.drain(dcb); passed");
 		}
@@ -427,6 +429,8 @@ private void drained() {
 		}
 
 		private DrainedData getDrainData() {
+			if (this.blob == null)
+				return getEmptyDrainData();
 			// System.out.println("**********************************");
 			DrainData dd = blob.getDrainData();
 			drainState = 5;

From cec98075699fc6447e2cbcbcff0ea268cb9a326b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 20 Mar 2014 12:19:25 +0800
Subject: [PATCH 187/881] Clears the BlobsManager Obj to get more heap space

BlobsManagerImpl keeps blobs and corecodes. We need to clear them from the
heap before making new blob. Otherwise there won't be enough heap space to make
new blob.
---
 .../distributed/node/CfgStringProcessorImpl.java     | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 827f13ea..bc3c94db 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -72,6 +72,14 @@ public void process(String json, ConfigType type, DrainData drainData) {
 		} else {
 			System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
 			System.out.println("New Configuration.....");
+			// [2014-3-20] We need to release blobsmanager to release the
+			// memory. Otherwise, Blobthread2.corecode will hold the memory.
+			BlobsManager bm = streamNode.getBlobsManager();
+			if (bm != null) {
+				bm.stop();
+				streamNode.setBlobsManager(null);
+				bm = null;
+			}
 			Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
 			ImmutableSet<Blob> blobSet = getBlobs(cfg, staticConfig, drainData);
 			if (blobSet != null) {
@@ -85,10 +93,6 @@ public void process(String json, ConfigType type, DrainData drainData) {
 				Map<Token, TCPConnectionInfo> conInfoMap = (Map<Token, TCPConnectionInfo>) cfg
 						.getExtraData(GlobalConstants.CONINFOMAP);
 
-				BlobsManager bm = streamNode.getBlobsManager();
-				if (bm != null)
-					bm.stop();
-
 				streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
 						conInfoMap, streamNode, conProvider));
 			} else {

From 4ad98e3451df148dd9d88c2651aebdea26a6b102 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 13:38:26 +0800
Subject: [PATCH 188/881] App name is passed to saveConfig()

---
 .../mit/streamjit/impl/common/ConfigurationEditor.java   | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 52518ccd..3c5db40e 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -214,7 +214,7 @@ private static void convert() {
 			String pythonDict = reader.readLine();
 
 			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
-			saveConfg(finalCfg, 0);
+			saveConfg(finalCfg, 0, name);
 		} catch (Exception ex) {
 			ex.printStackTrace();
 		}
@@ -291,20 +291,19 @@ private static String getConfigurationString(Configuration cfg) {
 	/**
 	 * Save the configuration.
 	 */
-	private static void saveConfg(Configuration config, int round) {
+	private static void saveConfg(Configuration config, int round, String appName) {
 		String json = config.toJson();
-		String name = "erer";
 		try {
 
 			File dir = new File(String.format("configurations%s%s",
-					File.separator, name));
+					File.separator, appName));
 			if (!dir.exists())
 				if (!dir.mkdirs()) {
 					System.err.println("Make directory failed");
 					return;
 				}
 
-			File file = new File(dir, String.format("%d%s.cfg", round, name));
+			File file = new File(dir, String.format("%d%s.cfg", round, appName));
 			FileWriter writer = new FileWriter(file, false);
 			writer.write(json);
 			writer.flush();

From f97c139b64832756e77f86efa8fe383168788e06 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 13:43:13 +0800
Subject: [PATCH 189/881] Asks for input if the current input is invalid

---
 .../impl/common/ConfigurationEditor.java      | 42 +++++++++++++------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 3c5db40e..b89acef1 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -157,22 +157,38 @@ private static void edit1(String name, int maxWor)
 					IntParameter.class);
 
 			if (wrkrMachine != null) {
-				System.out.println(wrkrMachine.getName() + " - "
-						+ wrkrMachine.getValue());
-				int val = Integer.parseInt(keyinreader.readLine());
-				builder.removeParameter(wrkrMachine.getName());
-				builder.addParameter(new SwitchParameter<Integer>(wrkrMachine
-						.getName(), Integer.class, val, wrkrMachine
-						.getUniverse()));
+				System.out.println(wrkrMachine.toString());
+				boolean isOk1 = false;
+				while (!isOk1) {
+					try {
+						int val = Integer.parseInt(keyinreader.readLine());
+						builder.removeParameter(wrkrMachine.getName());
+						builder.addParameter(new SwitchParameter<Integer>(
+								wrkrMachine.getName(), Integer.class, val,
+								wrkrMachine.getUniverse()));
+						isOk1 = true;
+					} catch (Exception ex) {
+						ex.printStackTrace();
+						isOk1 = false;
+					}
+				}
 			}
 
 			if (wrkrCut != null) {
-				System.out.println(wrkrCut.getName() + " - "
-						+ wrkrCut.getValue());
-				int val = Integer.parseInt(keyinreader.readLine());
-				builder.removeParameter(wrkrCut.getName());
-				builder.addParameter(new IntParameter(wrkrCut.getName(),
-						wrkrCut.getRange(), val));
+				System.out.println(wrkrCut.toString());
+				boolean isOk = false;
+				while (!isOk) {
+					try {
+						int val = Integer.parseInt(keyinreader.readLine());
+						builder.removeParameter(wrkrCut.getName());
+						builder.addParameter(new IntParameter(
+								wrkrCut.getName(), wrkrCut.getRange(), val));
+						isOk = true;
+					} catch (Exception ex) {
+						ex.printStackTrace();
+						isOk = false;
+					}
+				}
 			}
 		}
 

From f679f3fb8c932652b5caaeb2948007d11b132e5c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 13:45:08 +0800
Subject: [PATCH 190/881] generate2() generates default cfg of

Compiler2BlobFactory
---
 .../impl/common/ConfigurationEditor.java      | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index b89acef1..dc0fb8f0 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -130,6 +130,28 @@ private static void generate1(OneToOneElement<?, ?> stream) {
 		}
 	}
 
+	/**
+	 * Generates default cfg of {@link Compiler2BlobFactory}. No modification
+	 * done.
+	 * 
+	 * @param stream
+	 */
+	private static void generate2(OneToOneElement<?, ?> stream) {
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
+		BlobFactory bf = new Compiler2BlobFactory();
+		// BlobFactory bf = new DistributedBlobFactory(1);
+
+		Configuration cfg = bf.getDefaultConfiguration(Workers
+				.getAllWorkersInGraph(source));
+
+		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+
+		saveConfg(cfg, 0, name);
+
+	}
+
 	private static void edit1(String name, int maxWor)
 			throws NumberFormatException, IOException {
 		Configuration cfg;

From 7143d43e6b90dec0083741cf8e53384e988a390e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 13:47:14 +0800
Subject: [PATCH 191/881] noOfNodes is made as a method arg

---
 .../streamjit/impl/common/ConfigurationEditor.java    | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index dc0fb8f0..cbb5d38b 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,6 +16,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
@@ -36,15 +37,13 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		generate1(new FilterBank6.FilterBankPipeline());
+		generate1(new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
 		edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 	}
 
-	private static void generate(OneToOneElement<?, ?> stream) {
-		int noOfnodes = 4;
-
+	private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {
 		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
 		stream.visit(primitiveConnector);
 		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
@@ -102,9 +101,7 @@ private static void edit(String name, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
-	private static void generate1(OneToOneElement<?, ?> stream) {
-		int noOfnodes = 4;
-
+	private static void generate1(OneToOneElement<?, ?> stream, int noOfnodes) {
 		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
 		stream.visit(primitiveConnector);
 		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();

From 7c53feb2e229ea98142ff8fce58937d2cdf6d7d0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 15:29:16 +0800
Subject: [PATCH 192/881] Prints heap status for each reconfiguration

---
 .../impl/distributed/StreamJitAppManager.java       | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index d1ed76cf..2fdfe412 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -143,6 +143,18 @@ public boolean reconfigure(int multiplier) {
 			isRunning = false;
 		}
 
+		long heapMaxSize = Runtime.getRuntime().maxMemory();
+		long heapSize = Runtime.getRuntime().totalMemory();
+		long heapFreeSize = Runtime.getRuntime().freeMemory();
+
+		System.out.println("##############Controller######################");
+
+		System.out.println("heapMaxSize = " + heapMaxSize / 1e6);
+		System.out.println("heapSize = " + heapSize / 1e6);
+		System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
+		System.out.println("StraemJit app is running...");
+		System.out.println("##############################################");
+
 		return isRunning;
 	}
 
@@ -185,6 +197,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 						+ tailToken.toString(), 0, skipCount,
 				GlobalConstants.outputCount);
 	}
+
 	/**
 	 * Start the execution of the StreamJit application.
 	 */

From 6310e2fb849581d051d4e1a3e54a821d4c0df4da Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 21 Apr 2014 15:27:20 +0800
Subject: [PATCH 193/881] Renaming: tunerMode -> tunerStartMode

---
 .../mit/streamjit/impl/distributed/common/GlobalConstants.java  | 2 +-
 src/edu/mit/streamjit/tuner/TCPTuner.java                       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 674e7e8c..b7b682d8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -70,7 +70,7 @@ private GlobalConstants() {
 	 * tuning on remote machines.
 	 * </ol>
 	 */
-	public static final int tunerMode = 0;
+	public static final int tunerStartMode = 0;
 
 	/**
 	 * To turn on or turn off the drain data. If this is false, drain data will
diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index cb1b103e..b8dfbd1c 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -52,7 +52,7 @@ public void startTuner(String tunerPath) throws IOException {
 		int min = 5000;
 		Random rand = new Random();
 		Integer port = rand.nextInt(65535 - min) + min;
-		if (GlobalConstants.tunerMode == 0) {
+		if (GlobalConstants.tunerStartMode == 0) {
 			this.tuner = new ProcessBuilder("xterm", "-e", "python", tunerPath,
 					port.toString()).start();
 		} else

From 5dd901df5172ecfb8a92f44fc70adf931b055692 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 21 Apr 2014 16:43:04 +0800
Subject: [PATCH 194/881] Third option to start Opentuner

Controller starts python process silently. Not on any terminal.
---
 .../impl/distributed/common/GlobalConstants.java    | 13 ++++++++++---
 src/edu/mit/streamjit/tuner/TCPTuner.java           |  3 +++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index b7b682d8..18b43aca 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -61,10 +61,17 @@ private GlobalConstants() {
 	public static final String CONINFOMAP = "ConInfoMap";
 
 	/**
-	 * Whether to start the tuner automatically or not.
+	 * Decides how to start the opentuner. In first 2 cases, controller starts
+	 * opentuner and establishes connection with it on a random port no range
+	 * from 5000-65536. User can provide port no in 3 case.
+	 * 
 	 * <ol>
-	 * <li>0 - Controller will start the tuner automatically.
-	 * <li>1 - User has to manually start the tuner with correct portNo as
+	 * <li>0 - Controller starts the tuner automatically on a terminal. User can
+	 * see Opentuner related outputs in the new terminal.
+	 * <li>1 - Controller starts the tuner automatically as a Python process. No
+	 * explicit window will be opened. Suitable for remote running through SSH
+	 * terminal.
+	 * <li>2 - User has to manually start the tuner with correct portNo as
 	 * argument. Port no 12563 is used in this case. But it can be changed at
 	 * {@link TCPTuner#startTuner(String)}. We need this option to run the
 	 * tuning on remote machines.
diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index b8dfbd1c..494aef50 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -55,6 +55,9 @@ public void startTuner(String tunerPath) throws IOException {
 		if (GlobalConstants.tunerStartMode == 0) {
 			this.tuner = new ProcessBuilder("xterm", "-e", "python", tunerPath,
 					port.toString()).start();
+		} else if (GlobalConstants.tunerStartMode == 1) {
+			this.tuner = new ProcessBuilder("python", tunerPath,
+					port.toString()).start();
 		} else
 			port = 12563;
 		this.connection = new TunerConnection();

From d3e38ae9c13acef099678434e1a9adb6c86a5bfa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 12 May 2014 19:41:10 +0800
Subject: [PATCH 195/881] null multiplier case handled

IntParameter multiplier may not available in some configurations
( e.g., Interpreter mode). This case has been handled.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4b53fbf3..b2f84a25 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -163,8 +163,12 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				}
 			}
 
-			int multiplier = config.getParameter("multiplier",
-					IntParameter.class).getValue();
+			int multiplier = 1000;
+			IntParameter mulParam = config.getParameter("multiplier",
+					IntParameter.class);
+			if (mulParam != null)
+				multiplier = mulParam.getValue();
+
 			drainer.setBlobGraph(app.blobGraph);
 			System.err.println("Reconfiguring...multiplier = " + multiplier);
 			if (manager.reconfigure(multiplier)) {

From 0291bda304a8cd7147759a5bffc524e6a1066f93 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 08:54:06 +0800
Subject: [PATCH 196/881] Flushes the files for each iteration

Running time evaluation loops flush the file for each iteration.
---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java          | 1 +
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index c7a4f6df..ebe13a85 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -156,6 +156,7 @@ public void run() {
 
 					writer.write(time.toString());
 					writer.write('\n');
+					writer.flush();
 
 					reset();
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b2f84a25..040c236d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -251,6 +251,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 					ret = reconfigure(cfg);
 					writer.write(ret.second.toString());
 					writer.write('\n');
+					writer.flush();
 					total += ret.second;
 				}
 				double avg = total / count;

From 534e632a1e68e081e9d84e0bac9121d03e331313 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:01:35 +0800
Subject: [PATCH 197/881] Throws IllegalConfigurationException

Throws IllegalConfigurationException when the configuration file passed by the
user is invalid.
---
 .../distributed/DistributedStreamCompiler.java | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index f47a012f..0a09ba4b 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -149,7 +149,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 						.println("Reading the configuration from configuration file");
 				System.err
 						.println("No matching between parameters in the read configuration and parameters in the default configuration");
-				return null;
+				controller.closeAll();
+				throw new IllegalConfigurationException();
 			}
 			this.cfg = cfg1;
 		}
@@ -312,4 +313,19 @@ public void awaitDrained(long timeout, TimeUnit unit)
 			drainer.awaitDrained(timeout, unit);
 		}
 	}
+
+	private class IllegalConfigurationException extends RuntimeException {
+
+		private static final long serialVersionUID = 1L;
+
+		private static final String tag = "IllegalConfigurationException";
+
+		private IllegalConfigurationException() {
+			super(tag);
+		}
+
+		private IllegalConfigurationException(String msg) {
+			super(String.format("%s : %s", tag, msg));
+		}
+	}
 }
\ No newline at end of file

From ddbc05c3ee7d8a2a5482d0953a1ae073849bf84a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 5 Mar 2014 00:46:47 +0800
Subject: [PATCH 198/881] BlobNodes keep input output channel infos as well

We need this information to globally determine buffer sizes to avoid deadlocks.
This is added when implementing global buffer size adjustment.
---
 .../impl/common/AbstractDrainer.java          | 47 ++++++++++++++++++-
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index beadba0c..4f407c68 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -480,7 +480,7 @@ public BlobGraph(List<Set<Worker<?, ?>>> partitionWorkers) {
 
 			ImmutableMap.Builder<Token, BlobNode> builder = new ImmutableMap.Builder<>();
 			for (DummyBlob b : blobSet) {
-				builder.put(b.id, new BlobNode(b.id));
+				builder.put(b.id, new BlobNode(b.id, b.inputs, b.outputs));
 			}
 
 			this.blobNodes = builder.build();
@@ -524,6 +524,32 @@ public BlobNode getBlobNode(Token blobID) {
 			return blobNodes.get(blobID);
 		}
 
+		/**
+		 * TODO: We may need to make the class {@link BlobNode} public and move
+		 * these functions to {@link BlobNode}.
+		 * <p>
+		 * Returns output edges of a blob. This method is added on [2014-03-01].
+		 * 
+		 * @param blobID
+		 * @return
+		 */
+		public ImmutableSet<Token> getOutputs(Token blobID) {
+			return blobNodes.get(blobID).outputs;
+		}
+
+		/**
+		 * TODO: We may need to make the class {@link BlobNode} public and move
+		 * these functions to {@link BlobNode}.
+		 * <p>
+		 * Returns input edges of a blob. This method is added on [2014-03-01].
+		 * 
+		 * @param blobID
+		 * @return
+		 */
+		public ImmutableSet<Token> getInputs(Token blobID) {
+			return blobNodes.get(blobID).inputs;
+		}
+
 		/**
 		 * A Drainer can be set to the {@link BlobGraph} to perform draining.
 		 * 
@@ -650,12 +676,29 @@ private static final class BlobNode {
 		// TODO: add comments
 		private AtomicInteger drainState;
 
-		private BlobNode(Token blob) {
+		/**
+		 * All input channels of this blob. We need this information to globally
+		 * determine buffer sizes to avoid deadlocks. This is added on
+		 * [2014-03-01], when implementing global buffer size adjustment.
+		 */
+		private final ImmutableSet<Token> inputs;
+
+		/**
+		 * All output channels of this blob. We need this information to
+		 * globally determine buffer sizes to avoid deadlocks. This is added on
+		 * [2014-03-01], when implementing global buffer size adjustment.
+		 */
+		private final ImmutableSet<Token> outputs;
+
+		private BlobNode(Token blob, ImmutableSet<Token> inputs,
+				ImmutableSet<Token> outputs) {
 			this.blobID = blob;
 			predecessors = new ArrayList<>();
 			successors = new ArrayList<>();
 			dependencyCount = new AtomicInteger(0);
 			drainState = new AtomicInteger(0);
+			this.inputs = inputs;
+			this.outputs = outputs;
 		}
 
 		/**

From c981ade7e46d01bfdbeff9f2501baf64fdf678c4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 May 2014 13:25:14 +0800
Subject: [PATCH 199/881] Handled another type of Deadlock

A deadlock situation occurs when draining. A Down blob, that has more than one
upper blobs, cannot progress because some of its up blobs are drained and hence
no input on the corresponding input channels, and other upper blobs blocked at
their output channels as downblob is not consuming any data. So those
non-drained upper blobs are going to stuck forever.

Conflicts:
	src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
---
 .../distributed/node/BlobsManagerImpl.java    | 104 ++++++++++++++++--
 1 file changed, 95 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 07308b60..df510570 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -55,6 +55,8 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private MonitorBuffers monBufs;
 
+	private volatile DrainDeadLockHandler drainDeadLockHandler;
+
 	private final CTRLRDrainProcessor drainProcessor;
 
 	private final CommandProcessor cmdProcessor;
@@ -70,6 +72,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
+		this.drainDeadLockHandler = null;
 
 		bufferMap = createBufferMap(blobSet);
 
@@ -117,6 +120,9 @@ public void stop() {
 
 		if (monBufs != null)
 			monBufs.stopMonitoring();
+
+		if (drainDeadLockHandler != null)
+			drainDeadLockHandler.stopit();
 	}
 
 	// TODO: Buffer sizes, including head and tail buffers, must be optimized.
@@ -343,13 +349,15 @@ private void doDrain(boolean reqDrainData) {
 					bc.stop(3);
 			}
 
-			for (Thread t : inputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
+			// TODO: [2014-03-14] I commented following lines to avoid one dead
+			// lock case when draining. Deadlock 5 and 6.
+			// for (Thread t : inputChannelThreads) {
+			// try {
+			// t.join();
+			// } catch (InterruptedException e) {
+			// e.printStackTrace();
+			// }
+			// }
 			// System.out.println("Blob " + blobID + "Input thread's joined");
 			if (this.blob != null) {
 				DrainCallback dcb = new DrainCallback(this);
@@ -358,6 +366,20 @@ private void doDrain(boolean reqDrainData) {
 			}
 			// System.out.println("Blob " + blobID +
 			// "this.blob.drain(dcb); passed");
+
+			boolean isLastBlob = true;
+			for (BlobExecuter be : blobExecuters) {
+				if (be.drainState == 0) {
+					isLastBlob = false;
+					break;
+				}
+			}
+
+			if (isLastBlob && drainDeadLockHandler == null) {
+				System.out.println("****Starting DrainDeadLockHandler***");
+				drainDeadLockHandler = new DrainDeadLockHandler();
+				drainDeadLockHandler.start();
+			}
 		}
 
 		private void drained() {
@@ -422,9 +444,14 @@ private void drained() {
 				}
 			}
 
-			if (isLastBlob && monBufs != null)
-				monBufs.stopMonitoring();
+			if (isLastBlob) {
+				if (monBufs != null)
+					monBufs.stopMonitoring();
+
+				if (drainDeadLockHandler != null)
+					drainDeadLockHandler.stopit();
 
+			}
 			// printDrainedStatus();
 		}
 
@@ -771,4 +798,63 @@ public void stopMonitoring() {
 			this.interrupt();
 		}
 	}
+
+	private class DrainDeadLockHandler extends Thread {
+		final AtomicBoolean run;
+
+		private DrainDeadLockHandler() {
+			this.run = new AtomicBoolean(true);
+		}
+
+		public void run() {
+			try {
+				Thread.sleep(5000);
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+
+			System.out
+					.println("DrainDeadLockHandler is goint to clean buffers...");
+			boolean areAllDrained = false;
+
+			while (run.get()) {
+				areAllDrained = true;
+				for (BlobExecuter be : blobExecuters) {
+					if (be.drainState == 1 || be.drainState == 2) {
+						// System.out.println(be.blobID + " is not drained");
+						areAllDrained = false;
+						for (Token t : be.blob.getOutputs()) {
+							Buffer b = bufferMap.get(t);
+							int size = b.size();
+							if (size == 0)
+								continue;
+							System.out.println(String.format(
+									"Buffer %s has %d data. Going to clean it",
+									t.toString(), size));
+							Object[] obArray = new Object[size];
+							b.readAll(obArray);
+						}
+						for (Token t : be.blob.getInputs()) {
+							Buffer b = bufferMap.get(t);
+							int size = b.size();
+							if (size == 0)
+								continue;
+							System.out.println(String.format(
+									"Buffer %s has %d data. Going to clean it",
+									t.toString(), size));
+							Object[] obArray = new Object[size];
+							b.readAll(obArray);
+						}
+					}
+				}
+
+				if (areAllDrained)
+					break;
+			}
+		}
+
+		public void stopit() {
+			this.run.set(false);
+		}
+	}
 }

From e0ebf32b520c5d01dd09ef16f17def10658c86f3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 15 Mar 2014 13:51:24 +0800
Subject: [PATCH 200/881] Deadlockhandler's wait time has been increased.

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index df510570..697fb9ca 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -808,7 +808,7 @@ private DrainDeadLockHandler() {
 
 		public void run() {
 			try {
-				Thread.sleep(5000);
+				Thread.sleep(60000);
 			} catch (InterruptedException e) {
 				e.printStackTrace();
 			}

From c9b29f3f44e3fae648ef72ab63cd4cdd86731c66 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 15 Mar 2014 15:21:21 +0800
Subject: [PATCH 201/881] waits for all threads to terminate

before starting next tuning trial.
---
 .../distributed/node/BlobsManagerImpl.java    | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 697fb9ca..b55f558c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -124,7 +124,6 @@ public void stop() {
 		if (drainDeadLockHandler != null)
 			drainDeadLockHandler.stopit();
 	}
-
 	// TODO: Buffer sizes, including head and tail buffers, must be optimized.
 	// consider adding some tuning factor
 	private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
@@ -320,12 +319,28 @@ private void stop() {
 				bc.stop(true);
 			}
 
-			for (Thread t : blobThreads)
+			for (Thread t : blobThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+
+			for (Thread t : inputChannelThreads) {
 				try {
 					t.join();
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
+			}
+			for (Thread t : outputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
 
 			if (monBufs != null)
 				monBufs.stopMonitoring();

From b3bab8a9369053922af1c940e16494ea23fa4272 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 15 Mar 2014 15:25:00 +0800
Subject: [PATCH 202/881] Wakes up sleeping thread to stop running

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index b55f558c..3df95a20 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -825,7 +825,7 @@ public void run() {
 			try {
 				Thread.sleep(60000);
 			} catch (InterruptedException e) {
-				e.printStackTrace();
+				return;
 			}
 
 			System.out
@@ -870,6 +870,7 @@ public void run() {
 
 		public void stopit() {
 			this.run.set(false);
+			this.interrupt();
 		}
 	}
 }

From 0508369c20c8712ed95338406f050b0a2cb839c9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 May 2014 14:32:56 +0800
Subject: [PATCH 203/881] useDrainDeadlockHandler flag has been added.

We can turn on or off the DrainDeadlockHandler.

Conflicts:
	src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
---
 .../distributed/node/BlobsManagerImpl.java    | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 3df95a20..f7578005 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -63,6 +63,13 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final ImmutableMap<Token, Buffer> bufferMap;
 
+	/**
+	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
+	 * time dead lock. Otherwise dynamic buffer will be used for local buffers
+	 * to handled drain time data growth.
+	 */
+	private final boolean useDrainDeadLockHandler;
+
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, TCPConnectionInfo> conInfoMap, StreamNode streamNode,
 			TCPConnectionProvider conProvider) {
@@ -73,6 +80,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
 		this.drainDeadLockHandler = null;
+		this.useDrainDeadLockHandler = false;
 
 		bufferMap = createBufferMap(blobSet);
 
@@ -382,18 +390,20 @@ private void doDrain(boolean reqDrainData) {
 			// System.out.println("Blob " + blobID +
 			// "this.blob.drain(dcb); passed");
 
-			boolean isLastBlob = true;
-			for (BlobExecuter be : blobExecuters) {
-				if (be.drainState == 0) {
-					isLastBlob = false;
-					break;
+			if (useDrainDeadLockHandler) {
+				boolean isLastBlob = true;
+				for (BlobExecuter be : blobExecuters) {
+					if (be.drainState == 0) {
+						isLastBlob = false;
+						break;
+					}
 				}
-			}
 
-			if (isLastBlob && drainDeadLockHandler == null) {
-				System.out.println("****Starting DrainDeadLockHandler***");
-				drainDeadLockHandler = new DrainDeadLockHandler();
-				drainDeadLockHandler.start();
+				if (isLastBlob && drainDeadLockHandler == null) {
+					System.out.println("****Starting DrainDeadLockHandler***");
+					drainDeadLockHandler = new DrainDeadLockHandler();
+					drainDeadLockHandler.start();
+				}
 			}
 		}
 

From 967b851cddf5572fcc12dc89bfe5a68e40d9c89f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 17 Mar 2014 00:10:56 +0800
Subject: [PATCH 204/881] Sets name to thread

Name is set to DrainDeadLockHandler thread.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f7578005..5c0c4f90 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -828,6 +828,7 @@ private class DrainDeadLockHandler extends Thread {
 		final AtomicBoolean run;
 
 		private DrainDeadLockHandler() {
+			super("DrainDeadLockHandler");
 			this.run = new AtomicBoolean(true);
 		}
 

From 0b85e0f91e24b49ce9cc1d1014e9439c5e12374b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 17 Apr 2014 18:15:12 +0800
Subject: [PATCH 205/881] Missing changes after cherry picking

Conflicts:
	src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 5c0c4f90..201a5bb1 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -392,7 +392,7 @@ private void doDrain(boolean reqDrainData) {
 
 			if (useDrainDeadLockHandler) {
 				boolean isLastBlob = true;
-				for (BlobExecuter be : blobExecuters) {
+				for (BlobExecuter be : blobExecuters.values()) {
 					if (be.drainState == 0) {
 						isLastBlob = false;
 						break;
@@ -845,7 +845,7 @@ public void run() {
 
 			while (run.get()) {
 				areAllDrained = true;
-				for (BlobExecuter be : blobExecuters) {
+				for (BlobExecuter be : blobExecuters.values()) {
 					if (be.drainState == 1 || be.drainState == 2) {
 						// System.out.println(be.blobID + " is not drained");
 						areAllDrained = false;

From f35a3667d45cc60d3a15381be3366388f8d1904d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 30 May 2014 17:11:14 +0800
Subject: [PATCH 206/881] Bug Fix - stop() get called twice.

TCPInput and Output channel.stop() get called twice with different stopTypes.
This causes deadlock.
---
 .../streamjit/impl/distributed/node/TCPInputChannel.java  | 8 ++++++--
 .../streamjit/impl/distributed/node/TCPOutputChannel.java | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index 67084278..96da4752 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -145,7 +145,7 @@ public void receiveData() {
 		int bufFullCount = 0;
 		try {
 			Object obj = tcpConnection.readObject();
-			if(obj == null)	// [2014-03-15] Sometimes null is received.
+			if (obj == null) // [2014-03-15] Sometimes null is received.
 				return;
 			count++;
 
@@ -347,7 +347,11 @@ public int getOtherNodeID() {
 	@Override
 	public void stop(int type) {
 		assert 0 < type && type < 4 : "Undefined stop type";
-		this.stopType.set(type);
+		if (this.stopType.get() == 0) {
+			stopType.set(type);
+		} else if (debugLevel > 0) {
+			System.err.println("Stop has already been called.");
+		}
 	}
 
 	@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index 0a74705e..55530942 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -168,8 +168,12 @@ public final void stop(boolean isFinal) {
 		if (debugLevel > 0)
 			System.out.println(Thread.currentThread().getName()
 					+ " - stop request");
-		this.isFinal = isFinal;
-		this.stopFlag.set(true);
+		if (!this.stopFlag.get()) {
+			this.isFinal = isFinal;
+			this.stopFlag.set(true);
+		} else if (debugLevel > 0) {
+			System.err.println("Stop has already been called.");
+		}
 	}
 
 	/**

From 26f0a7ebae4e390c8411b0e1d3ce6136446c53f4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 May 2014 15:50:26 +0800
Subject: [PATCH 207/881] AsynchronousTCPconnection class added

Blobs can use this class to asynchronously send data to downstream blobs.
This class doesn't support asynchronous receive. It might be supported later if
Java.NIO2 gives good performance improvement.
---
 .../common/AsynchronousTCPConnection.java     | 139 ++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
new file mode 100644
index 00000000..5db13e12
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -0,0 +1,139 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousSocketChannel;
+import java.util.concurrent.Future;
+
+public class AsynchronousTCPConnection implements Connection {
+	private ObjectOutputStream ooStream = null;
+	private AsynchronousSocketChannel asyncSktChannel;
+
+	private MyByteArrayOutputStream bAos;
+
+	private boolean isconnected = false;
+	private final int resetCount;
+
+	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
+		this(asyncSktChannel, 5000);
+	}
+
+	/**
+	 * @param socket
+	 * @param resetCount
+	 *            reset the {@link ObjectOutputStream} after this no of sends.
+	 *            To avoid out of memory error.
+	 */
+	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
+			int resetCount) {
+		this.resetCount = resetCount;
+		try {
+			this.asyncSktChannel = asyncSktChannel;
+
+			bAos = new MyByteArrayOutputStream(4096);
+			ooStream = new ObjectOutputStream(bAos);
+			isconnected = true;
+			// System.out.println(String.format(
+			// "DEBUG: TCP connection %d has been established", count++));
+		} catch (IOException iex) {
+			isconnected = false;
+			iex.printStackTrace();
+		}
+	}
+
+	// This is introduced to reduce the ooStream.reset(); frequency. Too many
+	// resets, i.e., reset the ooStream for every new write severely affects the
+	// performance.
+	int n = 0;
+
+	@Override
+	public void writeObject(Object obj) throws IOException {
+		if (isStillConnected()) {
+			try {
+				ooStream.writeObject(obj);
+				ByteBuffer bb = ByteBuffer.wrap(bAos.getBuf(), 0,
+						bAos.getCount());
+
+				Future<Integer> nBytes = asyncSktChannel.write(bb);
+				bAos.reset();
+
+				n++;
+				// TODO: Any way to improve the performance?
+				if (n > resetCount) {
+					n = 0;
+					ooStream.reset();
+				}
+				// System.out.println("Object send...");
+			} catch (IOException ix) {
+				// Following doesn't change when other side of the socket is
+				// closed.....
+				/*
+				 * System.out.println("socket.isBound()" + socket.isBound());
+				 * System.out.println("socket.isClosed()" + socket.isClosed());
+				 * System.out.println("socket.isConnected()" +
+				 * socket.isConnected());
+				 * System.out.println("socket.isInputShutdown()" +
+				 * socket.isInputShutdown());
+				 * System.out.println("socket.isOutputShutdown()" +
+				 * socket.isOutputShutdown());
+				 */
+				isconnected = false;
+				throw ix;
+			}
+		} else {
+			throw new IOException("TCPConnection: Socket is not connected");
+		}
+	}
+
+	public final void closeConnection() {
+		try {
+			if (ooStream != null)
+				this.ooStream.close();
+			if (asyncSktChannel != null)
+				this.asyncSktChannel.close();
+		} catch (IOException ex) {
+			isconnected = false;
+			ex.printStackTrace();
+		}
+	}
+
+	@Override
+	public final boolean isStillConnected() {
+		// return (this.socket.isConnected() && !this.socket.isClosed());
+		return isconnected;
+	}
+
+	@Override
+	public <T> T readObject() throws IOException, ClassNotFoundException {
+		throw new IOException(
+				"Reading object is not supported in asynchronous tcp mode");
+	}
+
+	public InetAddress getInetAddress() {
+		throw new java.lang.Error("Method not Implemented");
+	}
+
+	@Override
+	public void softClose() throws IOException {
+		this.ooStream.write('\u001a');
+		this.ooStream.flush();
+	}
+
+	// http://stackoverflow.com/a/15686667/505406
+	private class MyByteArrayOutputStream extends ByteArrayOutputStream {
+		public MyByteArrayOutputStream(int size) {
+			super(size);
+		}
+
+		public int getCount() {
+			return count;
+		}
+
+		public byte[] getBuf() {
+			return buf;
+		}
+	}
+}

From 9dc9accc148de88b07b811d155684dfff97a5fa6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 May 2014 15:53:43 +0800
Subject: [PATCH 208/881] Connection factory returns Asynchronous cons

---
 .../distributed/common/ConnectionFactory.java | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
index e7946b94..23c87ac8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
@@ -1,8 +1,13 @@
 package edu.mit.streamjit.impl.distributed.common;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.nio.channels.AsynchronousServerSocketChannel;
+import java.nio.channels.AsynchronousSocketChannel;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 
 import edu.mit.streamjit.impl.distributed.runtimer.ListenerSocket;
 
@@ -79,4 +84,20 @@ public static Connection getConnection(Socket socket, boolean needSync)
 		else
 			return new TCPConnection(socket);
 	}
+
+	public static AsynchronousTCPConnection getAsyncConnection(
+			String serverAddress, int portNo) throws IOException,
+			InterruptedException, ExecutionException {
+		AsynchronousServerSocketChannel ssc;
+		AsynchronousSocketChannel sc2;
+		System.out.println("Inside initialization");
+		InetSocketAddress isa = new InetSocketAddress(serverAddress, portNo);
+
+		ssc = AsynchronousServerSocketChannel.open().bind(isa);
+		Future<AsynchronousSocketChannel> accepted = ssc.accept();
+		sc2 = accepted.get();
+
+		ssc.close();
+		return new AsynchronousTCPConnection(sc2);
+	}
 }

From d1ab6162705574ede886c295d979de5c3fa68825 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 12 May 2014 07:38:29 +0800
Subject: [PATCH 209/881] Bulk write in Asynchronous mode has been added

Writes an array of objects to the TCP output channel
---
 .../common/AsynchronousTCPConnection.java     | 58 +++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 5db13e12..10748683 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -6,13 +6,17 @@
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousSocketChannel;
+import java.nio.channels.CompletionHandler;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public class AsynchronousTCPConnection implements Connection {
 	private ObjectOutputStream ooStream = null;
 	private AsynchronousSocketChannel asyncSktChannel;
 
 	private MyByteArrayOutputStream bAos;
+	private AtomicBoolean canWrite;
 
 	private boolean isconnected = false;
 	private final int resetCount;
@@ -36,6 +40,7 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 			bAos = new MyByteArrayOutputStream(4096);
 			ooStream = new ObjectOutputStream(bAos);
 			isconnected = true;
+			canWrite = new AtomicBoolean(true);
 			// System.out.println(String.format(
 			// "DEBUG: TCP connection %d has been established", count++));
 		} catch (IOException iex) {
@@ -136,4 +141,57 @@ public byte[] getBuf() {
 			return buf;
 		}
 	}
+
+	public int write(Object[] data, int offset, int length) throws IOException,
+			InterruptedException, ExecutionException {
+
+		final MyByteArrayOutputStream bAos;
+		final ObjectOutputStream objOS;
+		final AtomicBoolean canWrite;
+
+		objOS = this.ooStream;
+		bAos = this.bAos;
+		canWrite = this.canWrite;
+
+		while (!canWrite.get())
+			Thread.sleep(10);
+
+		int written = 0;
+		while (written < length) {
+			objOS.writeObject(data[offset++]);
+			++written;
+		}
+
+		canWrite.set(false);
+		ByteBuffer bb = ByteBuffer.wrap(bAos.getBuf(), 0, bAos.getCount());
+		asyncSktChannel.write(bb, bb,
+				new CompletionHandler<Integer, ByteBuffer>() {
+					@Override
+					public void completed(Integer result, ByteBuffer attachment) {
+
+						if (attachment.hasRemaining()) {
+							asyncSktChannel.write(attachment, attachment, this);
+						} else {
+							System.out.println("Completed.. ");
+							canWrite.set(true);
+							bAos.reset();
+							try {
+								objOS.reset();
+							} catch (IOException e) {
+								// TODO Auto-generated catch block
+								e.printStackTrace();
+							}
+						}
+					}
+
+					@Override
+					public void failed(Throwable exc, ByteBuffer attachment) {
+						exc.printStackTrace();
+						System.out.println("**************************");
+						canWrite.set(false);
+					}
+				});
+
+		return written;
+	}
 }

From 3db289dfb8a30da731d40ea65b9c8e38ff366a99 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 12 May 2014 11:55:57 +0800
Subject: [PATCH 210/881] Makes asynchronousTCP connection

---
 .../impl/distributed/common/ConnectionFactory.java | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
index 23c87ac8..c07707c8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
@@ -85,17 +85,21 @@ public static Connection getConnection(Socket socket, boolean needSync)
 			return new TCPConnection(socket);
 	}
 
-	public static AsynchronousTCPConnection getAsyncConnection(
-			String serverAddress, int portNo) throws IOException,
-			InterruptedException, ExecutionException {
+	public static AsynchronousTCPConnection getAsyncConnection(int portNo)
+			throws IOException {
 		AsynchronousServerSocketChannel ssc;
 		AsynchronousSocketChannel sc2;
 		System.out.println("Inside initialization");
-		InetSocketAddress isa = new InetSocketAddress(serverAddress, portNo);
+		InetSocketAddress isa = new InetSocketAddress("", portNo);
 
 		ssc = AsynchronousServerSocketChannel.open().bind(isa);
 		Future<AsynchronousSocketChannel> accepted = ssc.accept();
-		sc2 = accepted.get();
+		try {
+			sc2 = accepted.get();
+		} catch (InterruptedException | ExecutionException ex) {
+			ex.printStackTrace();
+			return null;
+		}
 
 		ssc.close();
 		return new AsynchronousTCPConnection(sc2);

From e36e90c3f57fb36cb3568315f9037b46e51460bf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:04:18 +0800
Subject: [PATCH 211/881] Print message rearranged.

---
 .../streamjit/impl/distributed/common/ConnectionFactory.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
index c07707c8..d794b0a6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
@@ -89,11 +89,12 @@ public static AsynchronousTCPConnection getAsyncConnection(int portNo)
 			throws IOException {
 		AsynchronousServerSocketChannel ssc;
 		AsynchronousSocketChannel sc2;
-		System.out.println("Inside initialization");
 		InetSocketAddress isa = new InetSocketAddress("", portNo);
 
 		ssc = AsynchronousServerSocketChannel.open().bind(isa);
 		Future<AsynchronousSocketChannel> accepted = ssc.accept();
+		System.out.println("Waiting for asynchronous socket connection @ port "
+				+ portNo);
 		try {
 			sc2 = accepted.get();
 		} catch (InterruptedException | ExecutionException ex) {

From 964432ff043e2fc8f12976eb9f60db86eedd6e65 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:09:48 +0800
Subject: [PATCH 212/881] ByteBufferOutputStream has been added

ByteArrayOutputStream is modified to use ByteBuffer for performance.
---
 .../common/AsynchronousTCPConnection.java     | 337 ++++++++++++++++++
 1 file changed, 337 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 10748683..a44cfc1a 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -3,6 +3,8 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousSocketChannel;
@@ -194,4 +196,339 @@ public void failed(Throwable exc, ByteBuffer attachment) {
 
 		return written;
 	}
+
+	/**
+	 * This class implements an output stream in which the data is written into
+	 * a byte array. The buffer automatically grows as data is written to it.
+	 * The data can be retrieved using <code>toByteArray()</code> and
+	 * <code>toString()</code>.
+	 * <p>
+	 * Closing a <tt>ByteArrayOutputStream</tt> has no effect. The methods in
+	 * this class can be called after the stream has been closed without
+	 * generating an <tt>IOException</tt>.
+	 * 
+	 * @author Arthur van Hoff
+	 * @since JDK1.0
+	 * 
+	 * 
+	 *        This is just a copy of {@link ByteArrayOutputStream} and just byte
+	 *        array is replaced by {@link ByteBuffer} for performance.
+	 * @author sumanan
+	 * @since May 10, 2014
+	 */
+	public class ByteBufferOutputStream extends OutputStream {
+
+		/**
+		 * The buffer where data is stored.
+		 */
+		protected ByteBuffer bb;
+
+		/**
+		 * The number of valid bytes in the buffer.
+		 */
+		protected int count;
+
+		/**
+		 * Creates a new byte array output stream. The buffer capacity is
+		 * initially 32 bytes, though its size increases if necessary.
+		 */
+		public ByteBufferOutputStream() {
+			this(10 * 1024 * 1024);
+		}
+
+		public int getCount() {
+			return count;
+		}
+
+		/**
+		 * Creates a new byte array output stream, with a buffer capacity of the
+		 * specified size, in bytes.
+		 * 
+		 * @param size
+		 *            the initial size.
+		 * @exception IllegalArgumentException
+		 *                if size is negative.
+		 */
+		public ByteBufferOutputStream(int size) {
+			if (size < 0) {
+				throw new IllegalArgumentException("Negative initial size: "
+						+ size);
+			}
+			bb = ByteBuffer.allocateDirect(size);
+		}
+
+		/**
+		 * Increases the capacity if necessary to ensure that it can hold at
+		 * least the number of elements specified by the minimum capacity
+		 * argument.
+		 * 
+		 * @param minCapacity
+		 *            the desired minimum capacity
+		 * @throws OutOfMemoryError
+		 *             if {@code minCapacity < 0}. This is interpreted as a
+		 *             request for the unsatisfiably large capacity
+		 *             {@code (long) Integer.MAX_VALUE + (minCapacity - Integer.MAX_VALUE)}
+		 *             .
+		 */
+		private void ensureCapacity(int minCapacity) {
+			// overflow-conscious code
+			if (minCapacity - bb.capacity() > 0)
+				grow(minCapacity);
+		}
+
+		/**
+		 * Increases the capacity to ensure that it can hold at least the number
+		 * of elements specified by the minimum capacity argument.
+		 * 
+		 * @param minCapacity
+		 *            the desired minimum capacity
+		 */
+		private void grow(int minCapacity) {
+			// overflow-conscious code
+			int oldCapacity = bb.capacity();
+			int newCapacity = oldCapacity << 1;
+			if (newCapacity - minCapacity < 0)
+				newCapacity = minCapacity;
+			if (newCapacity < 0) {
+				if (minCapacity < 0) // overflow
+					throw new OutOfMemoryError();
+				newCapacity = Integer.MAX_VALUE;
+			}
+			ByteBuffer newBb = ByteBuffer.allocateDirect(newCapacity);
+			newBb.clear();
+			bb.flip();
+			newBb.put(bb);
+			bb = newBb;
+		}
+
+		/**
+		 * Writes <code>len</code> bytes from the specified byte array starting
+		 * at offset <code>off</code> to this output stream. The general
+		 * contract for <code>write(b, off, len)</code> is that some of the
+		 * bytes in the array <code>b</code> are written to the output stream in
+		 * order; element <code>b[off]</code> is the first byte written and
+		 * <code>b[off+len-1]</code> is the last byte written by this operation.
+		 * <p>
+		 * The <code>write</code> method of <code>OutputStream</code> calls the
+		 * write method of one argument on each of the bytes to be written out.
+		 * Subclasses are encouraged to override this method and provide a more
+		 * efficient implementation.
+		 * <p>
+		 * If <code>b</code> is <code>null</code>, a
+		 * <code>NullPointerException</code> is thrown.
+		 * <p>
+		 * If <code>off</code> is negative, or <code>len</code> is negative, or
+		 * <code>off+len</code> is greater than the length of the array
+		 * <code>b</code>, then an <tt>IndexOutOfBoundsException</tt> is thrown.
+		 * 
+		 * @param b
+		 *            the data.
+		 * @param off
+		 *            the start offset in the data.
+		 * @param len
+		 *            the number of bytes to write.
+		 * @exception IOException
+		 *                if an I/O error occurs. In particular, an
+		 *                <code>IOException</code> is thrown if the output
+		 *                stream is closed.
+		 */
+		public void write(byte b[], int off, int len) throws IOException {
+			if (b == null) {
+				throw new NullPointerException();
+			} else if ((off < 0) || (off > b.length) || (len < 0)
+					|| ((off + len) > b.length) || ((off + len) < 0)) {
+				throw new IndexOutOfBoundsException();
+			} else if (len == 0) {
+				return;
+			}
+			ensureCapacity(count + len);
+			bb.put(b, off, len);
+			count += len;
+			assert count == bb.position() : "count != bb.position()";
+		}
+
+		/**
+		 * Writes the specified byte to this byte array output stream.
+		 * 
+		 * @param b
+		 *            the byte to be written.
+		 */
+		public synchronized void write(int b) {
+			ensureCapacity(count + 1);
+			bb.put((byte) b);
+			count += 1;
+			assert count == bb.position() : "count != bb.position()";
+		}
+
+		/**
+		 * Writes <code>len</code> bytes from the specified byte array starting
+		 * at offset <code>off</code> to this byte array output stream.
+		 * 
+		 * @param b
+		 *            the data.
+		 * @param off
+		 *            the start offset in the data.
+		 * @param len
+		 *            the number of bytes to write.
+		 * 
+		 *            public synchronized void write(byte b[], int off, int len)
+		 *            { if ((off < 0) || (off > b.length) || (len < 0) || ((off
+		 *            + len) - b.length > 0)) { throw new
+		 *            IndexOutOfBoundsException(); } ensureCapacity(count +
+		 *            len); System.arraycopy(b, off, buf, count, len); count +=
+		 *            len; }
+		 */
+
+		/**
+		 * Writes the complete contents of this byte array output stream to the
+		 * specified output stream argument, as if by calling the output
+		 * stream's write method using <code>out.write(buf, 0, count)</code>.
+		 * 
+		 * @param out
+		 *            the output stream to which to write the data.
+		 * @exception IOException
+		 *                if an I/O error occurs.
+		 */
+		public synchronized void writeTo(OutputStream out) throws IOException {
+			out.write(getByteArray(), 0, count);
+		}
+
+		/**
+		 * Resets the <code>count</code> field of this byte array output stream
+		 * to zero, so that all currently accumulated output in the output
+		 * stream is discarded. The output stream can be used again, reusing the
+		 * already allocated buffer space.
+		 * 
+		 * @see java.io.ByteArrayInputStream#count
+		 */
+		public synchronized void reset() {
+			bb.position(0);
+			bb.limit(bb.capacity());
+			count = 0;
+		}
+
+		/**
+		 * Creates a newly allocated byte array. Its size is the current size of
+		 * this output stream and the valid contents of the buffer have been
+		 * copied into it.
+		 * 
+		 * @return the current contents of this output stream, as a byte array.
+		 * @see java.io.ByteArrayOutputStream#size()
+		 */
+		public synchronized byte toByteArray()[] {
+			return getByteArray();
+		}
+
+		/**
+		 * Returns the current size of the buffer.
+		 * 
+		 * @return the value of the <code>count</code> field, which is the
+		 *         number of valid bytes in this output stream.
+		 * @see java.io.ByteArrayOutputStream#count
+		 */
+		public synchronized int size() {
+			assert count == bb.position() : "count != bb.position()";
+			return count;
+		}
+
+		/**
+		 * Converts the buffer's contents into a string decoding bytes using the
+		 * platform's default character set. The length of the new
+		 * <tt>String</tt> is a function of the character set, and hence may not
+		 * be equal to the size of the buffer.
+		 * 
+		 * <p>
+		 * This method always replaces malformed-input and unmappable-character
+		 * sequences with the default replacement string for the platform's
+		 * default character set. The
+		 * {@linkplain java.nio.charset.CharsetDecoder} class should be used
+		 * when more control over the decoding process is required.
+		 * 
+		 * @return String decoded from the buffer's contents.
+		 * @since JDK1.1
+		 */
+		public synchronized String toString() {
+			return new String(getByteArray(), 0, count);
+		}
+
+		/**
+		 * Converts the buffer's contents into a string by decoding the bytes
+		 * using the specified {@link java.nio.charset.Charset charsetName}. The
+		 * length of the new <tt>String</tt> is a function of the charset, and
+		 * hence may not be equal to the length of the byte array.
+		 * 
+		 * <p>
+		 * This method always replaces malformed-input and unmappable-character
+		 * sequences with this charset's default replacement string. The
+		 * {@link java.nio.charset.CharsetDecoder} class should be used when
+		 * more control over the decoding process is required.
+		 * 
+		 * @param charsetName
+		 *            the name of a supported
+		 *            {@linkplain java.nio.charset.Charset </code>charset<code>}
+		 * @return String decoded from the buffer's contents.
+		 * @exception UnsupportedEncodingException
+		 *                If the named charset is not supported
+		 * @since JDK1.1
+		 */
+		public synchronized String toString(String charsetName)
+				throws UnsupportedEncodingException {
+			return new String(getByteArray(), 0, count, charsetName);
+		}
+
+		/**
+		 * Creates a newly allocated string. Its size is the current size of the
+		 * output stream and the valid contents of the buffer have been copied
+		 * into it. Each character <i>c</i> in the resulting string is
+		 * constructed from the corresponding element <i>b</i> in the byte array
+		 * such that: <blockquote>
+		 * 
+		 * <pre>
+		 * c == (char) (((hibyte &amp; 0xff) &lt;&lt; 8) | (b &amp; 0xff))
+		 * </pre>
+		 * 
+		 * </blockquote>
+		 * 
+		 * @deprecated This method does not properly convert bytes into
+		 *             characters. As of JDK&nbsp;1.1, the preferred way to do
+		 *             this is via the <code>toString(String enc)</code> method,
+		 *             which takes an encoding-name argument, or the
+		 *             <code>toString()</code> method, which uses the platform's
+		 *             default character encoding.
+		 * 
+		 * @param hibyte
+		 *            the high byte of each resulting Unicode character.
+		 * @return the current contents of the output stream, as a string.
+		 * @see java.io.ByteArrayOutputStream#size()
+		 * @see java.io.ByteArrayOutputStream#toString(String)
+		 * @see java.io.ByteArrayOutputStream#toString()
+		 */
+		@Deprecated
+		public synchronized String toString(int hibyte) {
+			return new String(getByteArray(), hibyte, 0, count);
+		}
+
+		/**
+		 * Closing a <tt>ByteArrayOutputStream</tt> has no effect. The methods
+		 * in this class can be called after the stream has been closed without
+		 * generating an <tt>IOException</tt>.
+		 * <p>
+		 * 
+		 */
+		public void close() throws IOException {
+		}
+
+		private byte[] getByteArray() {
+			bb.flip();
+			final int size = bb.remaining();
+			byte[] buf = new byte[size];
+			bb.get(buf, 0, size);
+			assert count == bb.position() : "count != bb.position()";
+			return buf;
+		}
+
+		public ByteBuffer getByteBuffer() {
+			return bb;
+		}
+	}
 }

From 37e39a8087dcf74dc62485fb560dc454843b4013 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:16:31 +0800
Subject: [PATCH 213/881] Uses ByteBufferOutputStream

---
 .../common/AsynchronousTCPConnection.java     | 34 +++++--------------
 1 file changed, 9 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index a44cfc1a..cb2b6e37 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -17,7 +17,7 @@ public class AsynchronousTCPConnection implements Connection {
 	private ObjectOutputStream ooStream = null;
 	private AsynchronousSocketChannel asyncSktChannel;
 
-	private MyByteArrayOutputStream bAos;
+	private ByteBufferOutputStream bBos;
 	private AtomicBoolean canWrite;
 
 	private boolean isconnected = false;
@@ -39,8 +39,8 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 		try {
 			this.asyncSktChannel = asyncSktChannel;
 
-			bAos = new MyByteArrayOutputStream(4096);
-			ooStream = new ObjectOutputStream(bAos);
+			bBos = new ByteBufferOutputStream();
+			ooStream = new ObjectOutputStream(bBos);
 			isconnected = true;
 			canWrite = new AtomicBoolean(true);
 			// System.out.println(String.format(
@@ -61,11 +61,10 @@ public void writeObject(Object obj) throws IOException {
 		if (isStillConnected()) {
 			try {
 				ooStream.writeObject(obj);
-				ByteBuffer bb = ByteBuffer.wrap(bAos.getBuf(), 0,
-						bAos.getCount());
+				ByteBuffer bb = bBos.getByteBuffer();
 
 				Future<Integer> nBytes = asyncSktChannel.write(bb);
-				bAos.reset();
+				bBos.reset();
 
 				n++;
 				// TODO: Any way to improve the performance?
@@ -129,30 +128,15 @@ public void softClose() throws IOException {
 		this.ooStream.flush();
 	}
 
-	// http://stackoverflow.com/a/15686667/505406
-	private class MyByteArrayOutputStream extends ByteArrayOutputStream {
-		public MyByteArrayOutputStream(int size) {
-			super(size);
-		}
-
-		public int getCount() {
-			return count;
-		}
-
-		public byte[] getBuf() {
-			return buf;
-		}
-	}
-
 	public int write(Object[] data, int offset, int length) throws IOException,
 			InterruptedException, ExecutionException {
 
-		final MyByteArrayOutputStream bAos;
+		final ByteBufferOutputStream bBos;
 		final ObjectOutputStream objOS;
 		final AtomicBoolean canWrite;
 
 		objOS = this.ooStream;
-		bAos = this.bAos;
+		bBos = this.bBos;
 		canWrite = this.canWrite;
 
 		while (!canWrite.get())
@@ -165,7 +149,7 @@ public int write(Object[] data, int offset, int length) throws IOException,
 		}
 
 		canWrite.set(false);
-		ByteBuffer bb = ByteBuffer.wrap(bAos.getBuf(), 0, bAos.getCount());
+		ByteBuffer bb = bBos.getByteBuffer();
 		asyncSktChannel.write(bb, bb,
 				new CompletionHandler<Integer, ByteBuffer>() {
 					@Override
@@ -176,7 +160,7 @@ public void completed(Integer result, ByteBuffer attachment) {
 						} else {
 							System.out.println("Completed.. ");
 							canWrite.set(true);
-							bAos.reset();
+							bBos.reset();
 							try {
 								objOS.reset();
 							} catch (IOException e) {

From 70ff5c5478ff0df09517fd5f8d897ff197a50f0a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:21:45 +0800
Subject: [PATCH 214/881] Refactored

send() method, holds common lines from write() and writeObject(), is added.
---
 .../common/AsynchronousTCPConnection.java     | 121 ++++++++----------
 1 file changed, 56 insertions(+), 65 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index cb2b6e37..c24ca885 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -10,7 +10,6 @@
 import java.nio.channels.AsynchronousSocketChannel;
 import java.nio.channels.CompletionHandler;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public class AsynchronousTCPConnection implements Connection {
@@ -58,34 +57,16 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 
 	@Override
 	public void writeObject(Object obj) throws IOException {
+
 		if (isStillConnected()) {
+
+			while (!canWrite.get())
+				;
+
 			try {
 				ooStream.writeObject(obj);
-				ByteBuffer bb = bBos.getByteBuffer();
-
-				Future<Integer> nBytes = asyncSktChannel.write(bb);
-				bBos.reset();
-
-				n++;
-				// TODO: Any way to improve the performance?
-				if (n > resetCount) {
-					n = 0;
-					ooStream.reset();
-				}
-				// System.out.println("Object send...");
+				send();
 			} catch (IOException ix) {
-				// Following doesn't change when other side of the socket is
-				// closed.....
-				/*
-				 * System.out.println("socket.isBound()" + socket.isBound());
-				 * System.out.println("socket.isClosed()" + socket.isClosed());
-				 * System.out.println("socket.isConnected()" +
-				 * socket.isConnected());
-				 * System.out.println("socket.isInputShutdown()" +
-				 * socket.isInputShutdown());
-				 * System.out.println("socket.isOutputShutdown()" +
-				 * socket.isOutputShutdown());
-				 */
 				isconnected = false;
 				throw ix;
 			}
@@ -94,72 +75,49 @@ public void writeObject(Object obj) throws IOException {
 		}
 	}
 
-	public final void closeConnection() {
-		try {
-			if (ooStream != null)
-				this.ooStream.close();
-			if (asyncSktChannel != null)
-				this.asyncSktChannel.close();
-		} catch (IOException ex) {
-			isconnected = false;
-			ex.printStackTrace();
-		}
-	}
-
-	@Override
-	public final boolean isStillConnected() {
-		// return (this.socket.isConnected() && !this.socket.isClosed());
-		return isconnected;
-	}
-
-	@Override
-	public <T> T readObject() throws IOException, ClassNotFoundException {
-		throw new IOException(
-				"Reading object is not supported in asynchronous tcp mode");
-	}
-
-	public InetAddress getInetAddress() {
-		throw new java.lang.Error("Method not Implemented");
-	}
-
-	@Override
-	public void softClose() throws IOException {
-		this.ooStream.write('\u001a');
-		this.ooStream.flush();
-	}
-
 	public int write(Object[] data, int offset, int length) throws IOException,
 			InterruptedException, ExecutionException {
 
-		final ByteBufferOutputStream bBos;
 		final ObjectOutputStream objOS;
 		final AtomicBoolean canWrite;
 
 		objOS = this.ooStream;
-		bBos = this.bBos;
 		canWrite = this.canWrite;
 
 		while (!canWrite.get())
-			Thread.sleep(10);
+			;
 
 		int written = 0;
 		while (written < length) {
 			objOS.writeObject(data[offset++]);
 			++written;
 		}
+		send();
+		return written;
+	}
+
+	private void send() {
+		final ObjectOutputStream objOS;
+		final AtomicBoolean canWrite;
+		final ByteBufferOutputStream bBos;
+
+		objOS = this.ooStream;
+		bBos = this.bBos;
+		canWrite = this.canWrite;
 
 		canWrite.set(false);
 		ByteBuffer bb = bBos.getByteBuffer();
+		bb.flip();
 		asyncSktChannel.write(bb, bb,
 				new CompletionHandler<Integer, ByteBuffer>() {
 					@Override
 					public void completed(Integer result, ByteBuffer attachment) {
 
 						if (attachment.hasRemaining()) {
+							// System.out.println("Re writing......");
 							asyncSktChannel.write(attachment, attachment, this);
 						} else {
-							System.out.println("Completed.. ");
-							canWrite.set(true);
+							// System.out.println("Completed.. ");
 							bBos.reset();
 							try {
 								objOS.reset();
@@ -167,6 +125,7 @@ public void completed(Integer result, ByteBuffer attachment) {
 								// TODO Auto-generated catch block
 								e.printStackTrace();
 							}
+							canWrite.set(true);
 						}
 					}
 
@@ -177,8 +136,40 @@ public void failed(Throwable exc, ByteBuffer attachment) {
 						canWrite.set(false);
 					}
 				});
+	}
 
-		return written;
+	public final void closeConnection() {
+		try {
+			if (ooStream != null)
+				this.ooStream.close();
+			if (asyncSktChannel != null)
+				this.asyncSktChannel.close();
+		} catch (IOException ex) {
+			isconnected = false;
+			ex.printStackTrace();
+		}
+	}
+
+	@Override
+	public final boolean isStillConnected() {
+		// return (this.socket.isConnected() && !this.socket.isClosed());
+		return isconnected;
+	}
+
+	@Override
+	public <T> T readObject() throws IOException, ClassNotFoundException {
+		throw new IOException(
+				"Reading object is not supported in asynchronous tcp mode");
+	}
+
+	public InetAddress getInetAddress() {
+		throw new java.lang.Error("Method not Implemented");
+	}
+
+	@Override
+	public void softClose() throws IOException {
+		this.ooStream.write('\u001a');
+		this.ooStream.flush();
 	}
 
 	/**

From bd0086c825cbfd8b19aef0ec24ce52313670a47f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:35:34 +0800
Subject: [PATCH 215/881] Softclose() added.

---
 .../common/AsynchronousTCPConnection.java      | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index c24ca885..b69dc0ab 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -10,6 +10,7 @@
 import java.nio.channels.AsynchronousSocketChannel;
 import java.nio.channels.CompletionHandler;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public class AsynchronousTCPConnection implements Connection {
@@ -168,8 +169,21 @@ public InetAddress getInetAddress() {
 
 	@Override
 	public void softClose() throws IOException {
-		this.ooStream.write('\u001a');
-		this.ooStream.flush();
+		while (!canWrite.get())
+			;
+		char c = '\u001a';
+		byte[] b = new byte[8];
+		b[0] = (byte) c;
+		b[1] = (byte) (c << 8);
+		ByteBuffer buffer = ByteBuffer.wrap(b);
+		Future<Integer> nBytes = asyncSktChannel.write(buffer);
+		int n = -1;
+		try {
+			n = nBytes.get();
+		} catch (InterruptedException | ExecutionException e1) {
+			e1.printStackTrace();
+		}
+		System.out.println("softClose- nBytes = " + n);
 	}
 
 	/**

From 5fa2295b5aed9e3e5ef5d4298015a6c31d6773ce Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:38:00 +0800
Subject: [PATCH 216/881] Cleanup

---
 .../distributed/common/AsynchronousTCPConnection.java    | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index b69dc0ab..6f830733 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -21,7 +21,6 @@ public class AsynchronousTCPConnection implements Connection {
 	private AtomicBoolean canWrite;
 
 	private boolean isconnected = false;
-	private final int resetCount;
 
 	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
 		this(asyncSktChannel, 5000);
@@ -35,7 +34,6 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
 	 */
 	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 			int resetCount) {
-		this.resetCount = resetCount;
 		try {
 			this.asyncSktChannel = asyncSktChannel;
 
@@ -43,19 +41,12 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 			ooStream = new ObjectOutputStream(bBos);
 			isconnected = true;
 			canWrite = new AtomicBoolean(true);
-			// System.out.println(String.format(
-			// "DEBUG: TCP connection %d has been established", count++));
 		} catch (IOException iex) {
 			isconnected = false;
 			iex.printStackTrace();
 		}
 	}
 
-	// This is introduced to reduce the ooStream.reset(); frequency. Too many
-	// resets, i.e., reset the ooStream for every new write severely affects the
-	// performance.
-	int n = 0;
-
 	@Override
 	public void writeObject(Object obj) throws IOException {
 

From 7f543f8d0640c8614e730b79e3c1b82d983efff6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:39:47 +0800
Subject: [PATCH 217/881] writeObject() method disabled

throws java.lang.Error("Method not Implemented") if called.
---
 .../common/AsynchronousTCPConnection.java     | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 6f830733..14c7d297 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -49,22 +49,16 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 
 	@Override
 	public void writeObject(Object obj) throws IOException {
-
-		if (isStillConnected()) {
-
-			while (!canWrite.get())
-				;
-
-			try {
-				ooStream.writeObject(obj);
-				send();
-			} catch (IOException ix) {
-				isconnected = false;
-				throw ix;
-			}
-		} else {
-			throw new IOException("TCPConnection: Socket is not connected");
-		}
+		throw new java.lang.Error("Method not Implemented");
+		/*
+		 * if (isStillConnected()) {
+		 * 
+		 * while (!canWrite.get()) ;
+		 * 
+		 * try { ooStream.writeObject(obj); send(); } catch (IOException ix) {
+		 * isconnected = false; throw ix; } } else { throw new
+		 * IOException("TCPConnection: Socket is not connected"); }
+		 */
 	}
 
 	public int write(Object[] data, int offset, int length) throws IOException,

From 84370ecef86080446a019979f74b471af4f1ad35 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 09:44:57 +0800
Subject: [PATCH 218/881] Uses Asynchronous buffers.

---
 .../distributed/node/BlobsManagerImpl.java    | 49 ++++++++++++++-----
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 201a5bb1..6c5257b6 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -17,6 +17,7 @@
 import com.google.common.collect.Sets;
 
 import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.AsyncTCPBuffer;
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
@@ -53,6 +54,8 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final TCPConnectionProvider conProvider;
 	private Map<Token, TCPConnectionInfo> conInfoMap;
 
+	private Set<Token> globalOutputTokens;
+
 	private MonitorBuffers monBufs;
 
 	private volatile DrainDeadLockHandler drainDeadLockHandler;
@@ -84,10 +87,6 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 
 		bufferMap = createBufferMap(blobSet);
 
-		for (Blob b : blobSet) {
-			b.installBuffers(bufferMap);
-		}
-
 		Set<Token> locaTokens = getLocalTokens(blobSet);
 		blobExecuters = new HashMap<>();
 		for (Blob b : blobSet) {
@@ -157,8 +156,8 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet());
 		Set<Token> globalInputTokens = Sets.difference(
 				minInputBufCapaciy.keySet(), localTokens);
-		Set<Token> globalOutputTokens = Sets.difference(
-				minOutputBufCapaciy.keySet(), localTokens);
+		globalOutputTokens = Sets.difference(minOutputBufCapaciy.keySet(),
+				localTokens);
 
 		for (Token t : localTokens) {
 			int bufSize = Math.max(minInputBufCapaciy.get(t),
@@ -171,10 +170,6 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}
 
-		for (Token t : globalOutputTokens) {
-			int bufSize = minOutputBufCapaciy.get(t);
-			addBuffer(t, bufSize, bufferMapBuilder);
-		}
 		return bufferMapBuilder.build();
 	}
 
@@ -245,8 +240,8 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 		ImmutableMap.Builder<Token, BoundaryOutputChannel> outputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : outputTokens) {
 			TCPConnectionInfo conInfo = conInfoMap.get(t);
-			outputChannelMap.put(t, new TCPOutputChannel(bufferMap.get(t),
-					conProvider, conInfo, t.toString(), 0));
+			outputChannelMap.put(t, new AsyncTCPOutputChannel(conProvider,
+					conInfo, t.toString(), 0));
 		}
 		return outputChannelMap.build();
 	}
@@ -311,6 +306,34 @@ private void start() {
 				outputChannelThreads.add(t);
 			}
 
+			for (Thread t : outputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+
+			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+					.builder();
+			for (Token t : blob.getInputs()) {
+				bufferMapBuilder.put(t, bufferMap.get(t));
+			}
+
+			for (Token t : blob.getOutputs()) {
+				if (globalOutputTokens.contains(t)) {
+					AsyncTCPOutputChannel asyChannel = (AsyncTCPOutputChannel) outputChannels
+							.get(t);
+					Buffer b = new AsyncTCPBuffer(asyChannel.getConnection());
+					bufferMapBuilder.put(t, b);
+				} else {
+					Buffer b = bufferMap.get(t);
+					bufferMapBuilder.put(t, b);
+				}
+			}
+
+			blob.installBuffers(bufferMapBuilder.build());
+
 			for (Thread t : blobThreads)
 				t.start();
 
@@ -756,6 +779,7 @@ else if (be.drainState == 0) {
 				}
 			}
 		}
+
 		public void requestStop() {
 			stopping = true;
 		}
@@ -767,6 +791,7 @@ private class MonitorBuffers extends Thread {
 		private final int id;
 		private final AtomicBoolean stopFlag;
 		int sleepTime = 25000;
+
 		MonitorBuffers() {
 			super("MonitorBuffers");
 			stopFlag = new AtomicBoolean(false);

From f5c137f3214cdb798d10ba3d9e689d1a1a4f5aa6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 10:30:19 +0800
Subject: [PATCH 219/881] TCPConnectionProvider makes asynchronousConnection

---
 .../distributed/common/TCPConnection.java     | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 40bd79db..9e9dd262 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -231,7 +231,7 @@ public String toString() {
 	 */
 	public static class TCPConnectionProvider {
 
-		private ConcurrentMap<TCPConnectionInfo, TCPConnection> allConnections;
+		private ConcurrentMap<TCPConnectionInfo, Connection> allConnections;
 
 		private final int myNodeID;
 
@@ -270,7 +270,7 @@ public Connection getConnection(TCPConnectionInfo conInfo)
 		 */
 		public Connection getConnection(TCPConnectionInfo conInfo, int timeOut)
 				throws SocketTimeoutException, IOException {
-			TCPConnection con = allConnections.get(conInfo);
+			Connection con = allConnections.get(conInfo);
 			if (con != null) {
 				if (con.isStillConnected()) {
 					return con;
@@ -281,8 +281,12 @@ public Connection getConnection(TCPConnectionInfo conInfo, int timeOut)
 			}
 
 			if (conInfo.getSrcID() == myNodeID) {
-				con = ConnectionFactory.getConnection(conInfo.getPortNo(),
-						timeOut, false);
+				if (myNodeID == 0)
+					con = ConnectionFactory.getConnection(conInfo.getPortNo(),
+							timeOut, false);
+				else
+					con = ConnectionFactory.getAsyncConnection(conInfo
+							.getPortNo());
 			} else if (conInfo.getDstID() == myNodeID) {
 				InetAddress ipAddress = iNetAddressMap.get(conInfo.getSrcID());
 				if (ipAddress.isLoopbackAddress())
@@ -297,8 +301,13 @@ public Connection getConnection(TCPConnectionInfo conInfo, int timeOut)
 		}
 
 		public void closeAllConnections() {
-			for (TCPConnection con : allConnections.values()) {
-				con.closeConnection();
+			for (Connection con : allConnections.values()) {
+				try {
+					con.closeConnection();
+				} catch (IOException e) {
+					// TODO Auto-generated catch block
+					e.printStackTrace();
+				}
 			}
 		}
 	}

From 52ea3cd020d41970ace460db90e867bd3e0bdf13 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 14:18:19 +0800
Subject: [PATCH 220/881] AsynchronousSeverSocket listens on IPADDR_ANY

---
 .../streamjit/impl/distributed/common/ConnectionFactory.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
index d794b0a6..ac9b3db5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
@@ -89,7 +89,7 @@ public static AsynchronousTCPConnection getAsyncConnection(int portNo)
 			throws IOException {
 		AsynchronousServerSocketChannel ssc;
 		AsynchronousSocketChannel sc2;
-		InetSocketAddress isa = new InetSocketAddress("", portNo);
+		InetSocketAddress isa = new InetSocketAddress("0.0.0.0", portNo);
 
 		ssc = AsynchronousServerSocketChannel.open().bind(isa);
 		Future<AsynchronousSocketChannel> accepted = ssc.accept();

From b40f7b51e1b117059f6888f3d0fe776133882297 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 14 May 2014 14:19:52 +0800
Subject: [PATCH 221/881] starts all channels first

BlobManaer start all channels related to the machine first and then
starts the blobs.
---
 .../impl/distributed/node/BlobsManagerImpl.java  | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 6c5257b6..41176ac0 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -105,6 +105,9 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 	 * manage all CPU and I/O threads those are related to the {@link Blob}s.
 	 */
 	public void start() {
+		for (BlobExecuter be : blobExecuters.values())
+			be.startChannels();
+
 		for (BlobExecuter be : blobExecuters.values())
 			be.start();
 
@@ -293,18 +296,21 @@ private BlobExecuter(Token t, Blob blob,
 			this.blobID = t;
 		}
 
-		private void start() {
-			for (BoundaryInputChannel bc : inputChannels.values()) {
+		private void startChannels() {
+			for (BoundaryOutputChannel bc : outputChannels.values()) {
 				Thread t = new Thread(bc.getRunnable(), bc.name());
 				t.start();
-				inputChannelThreads.add(t);
+				outputChannelThreads.add(t);
 			}
 
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
+			for (BoundaryInputChannel bc : inputChannels.values()) {
 				Thread t = new Thread(bc.getRunnable(), bc.name());
 				t.start();
-				outputChannelThreads.add(t);
+				inputChannelThreads.add(t);
 			}
+		}
+
+		private void start() {
 
 			for (Thread t : outputChannelThreads) {
 				try {

From b64164e08e9b975efab1603847a4f3ee4e7b709b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 May 2014 05:49:28 +0800
Subject: [PATCH 222/881] ByteBufferArrayOutputStream has been added

This class is an OutputStream that contains array of ByteBufferOutputStreams
and multiplexes the serialized data in to the array of bytebuffer output
streams. Purpose is while asynchronous threads working on a bytebuffer to send
the data on socket, user thread can serialize next set of objects into the
next buffer.
---
 .../common/AsynchronousTCPConnection.java     | 116 +++++++++++++++++-
 1 file changed, 112 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 14c7d297..1bd5ba88 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -9,9 +9,12 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousSocketChannel;
 import java.nio.channels.CompletionHandler;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class AsynchronousTCPConnection implements Connection {
 	private ObjectOutputStream ooStream = null;
@@ -70,8 +73,7 @@ public int write(Object[] data, int offset, int length) throws IOException,
 		objOS = this.ooStream;
 		canWrite = this.canWrite;
 
-		while (!canWrite.get())
-			;
+		while (!canWrite.get());
 
 		int written = 0;
 		while (written < length) {
@@ -154,8 +156,7 @@ public InetAddress getInetAddress() {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!canWrite.get())
-			;
+		while (!canWrite.get());
 		char c = '\u001a';
 		byte[] b = new byte[8];
 		b[0] = (byte) c;
@@ -505,4 +506,111 @@ public ByteBuffer getByteBuffer() {
 			return bb;
 		}
 	}
+
+	private enum Status {
+		canWrite, beingWritten, canRead, beingRead
+	}
+
+	public class ByteBufferArrayOutputStream extends OutputStream {
+
+		private final int debugPrint;
+
+		private int readIndex;
+		private int writeIndex;
+		private final ByteBufferOutputStream[] bytebufferArray;
+		private Map<Integer, AtomicReference<Status>> bufferStatus;
+
+		public ByteBufferArrayOutputStream(int listSize) {
+			debugPrint = 0;
+			writeIndex = 0;
+			readIndex = 0;
+			bytebufferArray = new ByteBufferOutputStream[listSize];
+			bufferStatus = new HashMap<>(listSize);
+			for (int i = 0; i < bytebufferArray.length; i++) {
+				bytebufferArray[i] = new ByteBufferOutputStream();
+				bufferStatus.put(i,
+						new AtomicReference<Status>(Status.canWrite));
+			}
+		}
+
+		@Override
+		public void write(int b) throws IOException {
+			bytebufferArray[writeIndex].write(b);
+		}
+
+		public void write(byte b[], int off, int len) throws IOException {
+			bytebufferArray[writeIndex].write(b, off, len);
+		}
+
+		public boolean newWrite() {
+			if (bufferStatus.get(writeIndex).compareAndSet(Status.canWrite,
+					Status.beingWritten)) {
+				if (debugPrint > 0)
+					System.out.println(Thread.currentThread().getName()
+							+ " : newWrite-canWrite : " + "writeIndex - "
+							+ writeIndex + ", readIndex - " + readIndex);
+				return true;
+			} else {
+				if (debugPrint > 0)
+					System.out.println(Thread.currentThread().getName()
+							+ " : newWrite-failed : " + "writeIndex - "
+							+ writeIndex + ", readIndex - " + readIndex);
+				return false;
+			}
+		}
+
+		public void writeCompleted() {
+			if (debugPrint > 0)
+				System.out.println(Thread.currentThread().getName()
+						+ " : writeCompleted : " + "writeIndex - " + writeIndex
+						+ ", readIndex - " + readIndex);
+			boolean ret = bufferStatus.get(writeIndex).compareAndSet(
+					Status.beingWritten, Status.canRead);
+			if (!ret)
+				throw new IllegalStateException("bufferStatus conflict");
+
+			writeIndex = (writeIndex + 1) % bytebufferArray.length;
+		}
+
+		public synchronized ByteBufferOutputStream newRead() {
+			if (bufferStatus.get(readIndex).get() == Status.beingRead) {
+				if (debugPrint > 0)
+					System.out.println(Thread.currentThread().getName()
+							+ " : newRead-beingRead : " + "writeIndex - "
+							+ writeIndex + ", readIndex - " + readIndex);
+				return null;
+			}
+
+			if (bufferStatus.get(readIndex).compareAndSet(Status.canRead,
+					Status.beingRead)) {
+				if (debugPrint > 0)
+					System.out.println(Thread.currentThread().getName()
+							+ " : newRead-canRead : " + "writeIndex - "
+							+ writeIndex + ", readIndex - " + readIndex);
+				if (bytebufferArray[readIndex].getCount() == 0) {
+					throw new IllegalStateException(
+							"bytebufferArray[a].getCount() != 0 is expected.");
+				}
+				return bytebufferArray[readIndex];
+			} else {
+				if (debugPrint > 0)
+					System.out.println(Thread.currentThread().getName()
+							+ " : newRead - not can read " + readIndex);
+				return null;
+			}
+		}
+
+		public void readCompleted() {
+			if (debugPrint > 0)
+				System.out.println(Thread.currentThread().getName()
+						+ " : readCompleted : " + "writeIndex - " + writeIndex
+						+ ", readIndex - " + readIndex);
+			boolean ret = bufferStatus.get(readIndex).compareAndSet(
+					Status.beingRead, Status.canWrite);
+			if (!ret)
+				throw new IllegalStateException("bufferStatus conflict");
+			bytebufferArray[readIndex].reset();
+			readIndex = (readIndex + 1) % bytebufferArray.length;
+		}
+	}
 }

From 9435d00bea16dbe77ad322b96063d137f47ac26a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 May 2014 06:01:17 +0800
Subject: [PATCH 223/881] Prints msg when buffer grows.

---
 .../distributed/common/AsynchronousTCPConnection.java     | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 1bd5ba88..fc6117d4 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -185,9 +185,9 @@ public void softClose() throws IOException {
 	 * @author Arthur van Hoff
 	 * @since JDK1.0
 	 * 
-	 * 
-	 *        This is just a copy of {@link ByteArrayOutputStream} and just byte
-	 *        array is replaced by {@link ByteBuffer} for performance.
+	 *        This is a copy of {@link ByteArrayOutputStream} and byte array in
+	 *        ByteArrayOutputStream is replaced by {@link ByteBuffer} for
+	 *        performance.
 	 * @author sumanan
 	 * @since May 10, 2014
 	 */
@@ -274,6 +274,8 @@ private void grow(int minCapacity) {
 			bb.flip();
 			newBb.put(bb);
 			bb = newBb;
+			System.out.println("Growing bytebuffer. newCapacity = "
+					+ newCapacity);
 		}
 
 		/**

From 047664a879c6fbfdcec497647f5947db12a5437e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 May 2014 06:08:47 +0800
Subject: [PATCH 224/881] Uses ByteBufferArrayOutputStream to send data

Bulk write method uses ByteBufferArrayOutputStream to send data.
---
 .../common/AsynchronousTCPConnection.java     | 57 +++++++------------
 1 file changed, 21 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index fc6117d4..a0022930 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -13,16 +13,12 @@
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 public class AsynchronousTCPConnection implements Connection {
 	private ObjectOutputStream ooStream = null;
 	private AsynchronousSocketChannel asyncSktChannel;
-
-	private ByteBufferOutputStream bBos;
-	private AtomicBoolean canWrite;
-
+	private ByteBufferArrayOutputStream bBAos;
 	private boolean isconnected = false;
 
 	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
@@ -40,10 +36,9 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 		try {
 			this.asyncSktChannel = asyncSktChannel;
 
-			bBos = new ByteBufferOutputStream();
-			ooStream = new ObjectOutputStream(bBos);
+			bBAos = new ByteBufferArrayOutputStream(2);
+			ooStream = new ObjectOutputStream(bBAos);
 			isconnected = true;
-			canWrite = new AtomicBoolean(true);
 		} catch (IOException iex) {
 			isconnected = false;
 			iex.printStackTrace();
@@ -67,33 +62,33 @@ public void writeObject(Object obj) throws IOException {
 	public int write(Object[] data, int offset, int length) throws IOException,
 			InterruptedException, ExecutionException {
 
-		final ObjectOutputStream objOS;
-		final AtomicBoolean canWrite;
-
-		objOS = this.ooStream;
-		canWrite = this.canWrite;
-
-		while (!canWrite.get());
+		final ObjectOutputStream objOS = this.ooStream;
+		final ByteBufferArrayOutputStream bBAos = this.bBAos;
 
 		int written = 0;
-		while (written < length) {
-			objOS.writeObject(data[offset++]);
-			++written;
+		if (bBAos.newWrite()) {
+			while (written < length) {
+				objOS.writeObject(data[offset++]);
+				++written;
+			}
+			objOS.reset();
+			bBAos.writeCompleted();
 		}
+
 		send();
 		return written;
 	}
 
 	private void send() {
-		final ObjectOutputStream objOS;
-		final AtomicBoolean canWrite;
 		final ByteBufferOutputStream bBos;
+		final ByteBufferArrayOutputStream bBAos;
+
+		bBAos = this.bBAos;
 
-		objOS = this.ooStream;
-		bBos = this.bBos;
-		canWrite = this.canWrite;
+		bBos = bBAos.newRead();
+		if (bBos == null)
+			return;
 
-		canWrite.set(false);
 		ByteBuffer bb = bBos.getByteBuffer();
 		bb.flip();
 		asyncSktChannel.write(bb, bb,
@@ -102,18 +97,10 @@ private void send() {
 					public void completed(Integer result, ByteBuffer attachment) {
 
 						if (attachment.hasRemaining()) {
-							// System.out.println("Re writing......");
 							asyncSktChannel.write(attachment, attachment, this);
 						} else {
-							// System.out.println("Completed.. ");
-							bBos.reset();
-							try {
-								objOS.reset();
-							} catch (IOException e) {
-								// TODO Auto-generated catch block
-								e.printStackTrace();
-							}
-							canWrite.set(true);
+							bBAos.readCompleted();
+							send();
 						}
 					}
 
@@ -121,7 +108,6 @@ public void completed(Integer result, ByteBuffer attachment) {
 					public void failed(Throwable exc, ByteBuffer attachment) {
 						exc.printStackTrace();
 						System.out.println("**************************");
-						canWrite.set(false);
 					}
 				});
 	}
@@ -156,7 +142,6 @@ public InetAddress getInetAddress() {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!canWrite.get());
 		char c = '\u001a';
 		byte[] b = new byte[8];
 		b[0] = (byte) c;

From a03bc2bb91b6906a51001cab0985687561dc8298 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 May 2014 06:13:27 +0800
Subject: [PATCH 225/881] SoftClose() implemented using...

...ByteBufferArrayOutputStream
---
 .../common/AsynchronousTCPConnection.java     | 20 ++++++-------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index a0022930..ff393c36 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -12,7 +12,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicReference;
 
 public class AsynchronousTCPConnection implements Connection {
@@ -142,19 +141,12 @@ public InetAddress getInetAddress() {
 
 	@Override
 	public void softClose() throws IOException {
-		char c = '\u001a';
-		byte[] b = new byte[8];
-		b[0] = (byte) c;
-		b[1] = (byte) (c << 8);
-		ByteBuffer buffer = ByteBuffer.wrap(b);
-		Future<Integer> nBytes = asyncSktChannel.write(buffer);
-		int n = -1;
-		try {
-			n = nBytes.get();
-		} catch (InterruptedException | ExecutionException e1) {
-			e1.printStackTrace();
-		}
-		System.out.println("softClose- nBytes = " + n);
+		while (!bBAos.newWrite());
+		byte[] ba = "close\n".getBytes();
+		bBAos.bytebufferArray[bBAos.writeIndex].write(ba, 0, ba.length);
+		bBAos.writeCompleted();
+		send();
+		System.err.println("Softclosed is called");
 	}
 
 	/**

From 1daeddcf7d0d77b55a4b74de69515700d0071888 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 May 2014 06:46:13 +0800
Subject: [PATCH 226/881] Async TCP related 2 new classes have been added

---
 .../streamjit/impl/blob/AsyncTCPBuffer.java   | 45 ++++++++++
 .../node/AsyncTCPOutputChannel.java           | 85 +++++++++++++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java

diff --git a/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java b/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
new file mode 100644
index 00000000..763d1aa7
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
@@ -0,0 +1,45 @@
+package edu.mit.streamjit.impl.blob;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
+
+public class AsyncTCPBuffer extends AbstractWriteOnlyBuffer {
+
+	private final AsynchronousTCPConnection con;
+
+	public AsyncTCPBuffer(AsynchronousTCPConnection con) {
+		this.con = con;
+	}
+
+	@Override
+	public boolean write(Object t) {
+		try {
+			con.writeObject(t);
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return false;
+	}
+
+	public int write(Object[] data, int offset, int length) {
+		try {
+			return con.write(data, offset, length);
+		} catch (IOException | InterruptedException | ExecutionException e) {
+			e.printStackTrace();
+		}
+		return 0;
+	}
+
+	@Override
+	public int size() {
+		return 0;
+	}
+
+	@Override
+	public int capacity() {
+		return Integer.MAX_VALUE;
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
new file mode 100644
index 00000000..38417110
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -0,0 +1,85 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableList;
+
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
+import edu.mit.streamjit.impl.distributed.common.Connection;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+
+public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
+
+	private Connection con;
+
+	private final String name;
+
+	private final TCPConnectionProvider conProvider;
+
+	TCPConnectionInfo conInfo;
+
+	public AsyncTCPOutputChannel(TCPConnectionProvider conProvider,
+			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+		name = "AsyncTCPOutputChannel " + bufferTokenName;
+		this.conProvider = conProvider;
+		this.conInfo = conInfo;
+	}
+
+	@Override
+	public String name() {
+		return name;
+	}
+
+	@Override
+	public final void closeConnection() throws IOException {
+		con.softClose();
+	}
+
+	@Override
+	public boolean isStillConnected() {
+		return (con == null) ? false : con.isStillConnected();
+	}
+
+	@Override
+	public Runnable getRunnable() {
+		return new Runnable() {
+			@Override
+			public void run() {
+				if (con == null || !con.isStillConnected()) {
+					try {
+						con = conProvider.getConnection(conInfo);
+					} catch (IOException e) {
+						e.printStackTrace();
+					}
+				}
+			}
+		};
+	}
+
+	@Override
+	public int getOtherNodeID() {
+		return 0;
+	}
+
+	@Override
+	public ImmutableList<Object> getUnprocessedData() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public void stop(boolean isFinal) {
+
+	}
+
+	@Override
+	public void sendData() {
+
+	}
+
+	public AsynchronousTCPConnection getConnection() {
+		return (AsynchronousTCPConnection) con;
+	}
+}

From 12ff40189c5e8f87c69def52ebd96b11e06d4a72 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 19 May 2014 06:48:46 +0800
Subject: [PATCH 227/881] SoftClose() writes into OOStream

SoftClose() writes a byte to Object Output Stream which causes
OptionalDataException at receiver side.
---
 .../impl/distributed/common/AsynchronousTCPConnection.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index ff393c36..95d7aab6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -142,8 +142,8 @@ public InetAddress getInetAddress() {
 	@Override
 	public void softClose() throws IOException {
 		while (!bBAos.newWrite());
-		byte[] ba = "close\n".getBytes();
-		bBAos.bytebufferArray[bBAos.writeIndex].write(ba, 0, ba.length);
+		this.ooStream.write('\u001a');
+		this.ooStream.flush();
 		bBAos.writeCompleted();
 		send();
 		System.err.println("Softclosed is called");

From 8aff2f2e2bee3ad3086ce195730fae77ff037c65 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 19 May 2014 12:41:41 +0800
Subject: [PATCH 228/881] Exception -> Error

Method not Implemented error thrown
---
 .../impl/distributed/common/AsynchronousTCPConnection.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 95d7aab6..14f4fb25 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -131,7 +131,7 @@ public final boolean isStillConnected() {
 
 	@Override
 	public <T> T readObject() throws IOException, ClassNotFoundException {
-		throw new IOException(
+		throw new java.lang.Error(
 				"Reading object is not supported in asynchronous tcp mode");
 	}
 
@@ -146,7 +146,7 @@ public void softClose() throws IOException {
 		this.ooStream.flush();
 		bBAos.writeCompleted();
 		send();
-		System.err.println("Softclosed is called");
+		System.err.println("Softclose is called");
 	}
 
 	/**

From 935f0293774338c38500e6a01b6437f731906c14 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 19 May 2014 12:48:14 +0800
Subject: [PATCH 229/881] Synchronization bug fix

In readCompleted(), status is set to beingRead from canWrite before increasing
readIndex. This caused synchronization problem. This is fixed and the same
problem at writeCompleted() also fixed.
---
 .../common/AsynchronousTCPConnection.java         | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 14f4fb25..8ef7ebf3 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -543,12 +543,12 @@ public void writeCompleted() {
 				System.out.println(Thread.currentThread().getName()
 						+ " : writeCompleted : " + "writeIndex - " + writeIndex
 						+ ", readIndex - " + readIndex);
-			boolean ret = bufferStatus.get(writeIndex).compareAndSet(
+			int w = writeIndex;
+			writeIndex = (writeIndex + 1) % bytebufferArray.length;
+			boolean ret = bufferStatus.get(w).compareAndSet(
 					Status.beingWritten, Status.canRead);
 			if (!ret)
 				throw new IllegalStateException("bufferStatus conflict");
-
-			writeIndex = (writeIndex + 1) % bytebufferArray.length;
 		}
 
 		public synchronized ByteBufferOutputStream newRead() {
@@ -584,12 +584,13 @@ public void readCompleted() {
 				System.out.println(Thread.currentThread().getName()
 						+ " : readCompleted : " + "writeIndex - " + writeIndex
 						+ ", readIndex - " + readIndex);
-			boolean ret = bufferStatus.get(readIndex).compareAndSet(
-					Status.beingRead, Status.canWrite);
-			if (!ret)
-				throw new IllegalStateException("bufferStatus conflict");
 			bytebufferArray[readIndex].reset();
+			int r = readIndex;
 			readIndex = (readIndex + 1) % bytebufferArray.length;
+			boolean ret = bufferStatus.get(r).compareAndSet(Status.beingRead,
+					Status.canWrite);
+			if (!ret)
+				throw new IllegalStateException("bufferStatus conflict");
 		}
 	}
 }

From 00eb57adfed9ade420fed4402edbacb087931232 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 21 May 2014 05:46:20 +0800
Subject: [PATCH 230/881] Sets isConnected true if asy write failed

If CompletionHandler.failed() called, isConnected set to false.
---
 .../impl/distributed/common/AsynchronousTCPConnection.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 8ef7ebf3..eb8bca6b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -105,8 +105,8 @@ public void completed(Integer result, ByteBuffer attachment) {
 
 					@Override
 					public void failed(Throwable exc, ByteBuffer attachment) {
+						isconnected = false;
 						exc.printStackTrace();
-						System.out.println("**************************");
 					}
 				});
 	}

From 4ce0a3ca689758b37809351e940b139537b8b689 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 21 May 2014 06:26:49 +0800
Subject: [PATCH 231/881] AsyncTCPOutputChannel.stop() implemented

---
 .../distributed/node/AsyncTCPOutputChannel.java  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 38417110..89f273db 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -20,11 +20,17 @@ public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
 	TCPConnectionInfo conInfo;
 
+	private volatile boolean isFinal;
+
+	private volatile boolean stopCalled;
+
 	public AsyncTCPOutputChannel(TCPConnectionProvider conProvider,
 			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		name = "AsyncTCPOutputChannel " + bufferTokenName;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
+		isFinal = false;
+		stopCalled = false;
 	}
 
 	@Override
@@ -71,7 +77,15 @@ public ImmutableList<Object> getUnprocessedData() {
 
 	@Override
 	public void stop(boolean isFinal) {
-
+		this.isFinal = isFinal;
+		if (!stopCalled) {
+			try {
+				con.softClose();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		stopCalled = true;
 	}
 
 	@Override

From 4ceb5187d45f9603d5291931ef0c8eaaffe4f6e8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 21 May 2014 06:50:14 +0800
Subject: [PATCH 232/881] ConnectionInfo equals() and hashCode() treats

srcID and dstID as not interchangable. We need this scenario for
asymmetric connections like AsynchronousTCPConnection in which only either
reading or writing is supported.
---
 .../impl/distributed/common/Connection.java   | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 11cb6189..26a1bf0f 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -56,7 +56,7 @@ public interface Connection {
 	 * when the thread is blocked at {@link ObjectInputStream#readObject()}
 	 * method call.
 	 * </p>
-	 *
+	 * 
 	 * @throws IOException
 	 */
 	public void softClose() throws IOException;
@@ -105,10 +105,13 @@ public int getDstID() {
 		public int hashCode() {
 			final int prime = 31;
 			int result = 1;
-			int min = Math.min(srcID, dstID);
-			int max = Math.max(srcID, dstID);
-			result = prime * result + min;
-			result = prime * result + max;
+			/*
+			 * int min = Math.min(srcID, dstID); int max = Math.max(srcID,
+			 * dstID); result = prime * result + min; result = prime * result +
+			 * max;
+			 */
+			result = prime * result + srcID;
+			result = prime * result + dstID;
 			return result;
 		}
 
@@ -121,13 +124,15 @@ public boolean equals(Object obj) {
 			if (!(obj instanceof ConnectionInfo))
 				return false;
 			ConnectionInfo other = (ConnectionInfo) obj;
-			int myMin = Math.min(srcID, dstID);
-			int myMax = Math.max(srcID, dstID);
-			int otherMin = Math.min(other.srcID, other.dstID);
-			int otherMax = Math.max(other.srcID, other.dstID);
-			if (myMin != otherMin)
+			/*
+			 * int myMin = Math.min(srcID, dstID); int myMax = Math.max(srcID,
+			 * dstID); int otherMin = Math.min(other.srcID, other.dstID); int
+			 * otherMax = Math.max(other.srcID, other.dstID); if (myMin !=
+			 * otherMin) return false; if (myMax != otherMax) return false;
+			 */
+			if (srcID != other.srcID)
 				return false;
-			if (myMax != otherMax)
+			if (dstID != other.dstID)
 				return false;
 			return true;
 		}

From f67fa46dc21c23da5560fd87eef842632cb5fcb1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 08:45:30 +0800
Subject: [PATCH 233/881] Comments have been added to AsyTCPCon.

---
 .../common/AsynchronousTCPConnection.java     | 78 +++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index eb8bca6b..31a00167 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -14,10 +14,31 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
+/**
+ * Uses {@link AsynchronousSocketChannel} from Java's NIO.2 to send data. This
+ * class only supports bulk asynchronous write. Reads ({@link #readObject()}) or
+ * single object writes ({@link #writeObject(Object)}) are not supported.
+ * Serialises object array into {@link ByteBuffer} and sends it over a
+ * {@link AsynchronousSocketChannel}. Further, for the performance purposes, in
+ * oder to parallelise serialisation task and sending task, multiple
+ * {@link ByteBuffer}s are used. So that while user thread is serialising the
+ * data into a {@link ByteBuffer}, Java threads can send the already written
+ * bytebuffers.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 05, 2014
+ * 
+ */
 public class AsynchronousTCPConnection implements Connection {
+	/**
+	 * Backed by {@link ByteBufferArrayOutputStream}.
+	 */
 	private ObjectOutputStream ooStream = null;
+
 	private AsynchronousSocketChannel asyncSktChannel;
+
 	private ByteBufferArrayOutputStream bBAos;
+
 	private boolean isconnected = false;
 
 	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
@@ -486,17 +507,51 @@ public ByteBuffer getByteBuffer() {
 		}
 	}
 
+	/**
+	 * A {@link ByteBufferOutputStream} ( implicitly {@link ByteBuffer} ) can be
+	 * in one of following 4 state. State of a {@link ByteBufferOutputStream}
+	 * expected to change in a cyclic manner, from canWrite -> beingWritten ->
+	 * canRead -> beingRead -> canWrite.
+	 * 
+	 * @author sumanan
+	 */
 	private enum Status {
 		canWrite, beingWritten, canRead, beingRead
 	}
 
+	/**
+	 * Writers must call {@link #newWrite()} before begins the write process and
+	 * call {@link #writeCompleted()} after the end of write process. Whatever
+	 * written in between these two calls will be captured into single
+	 * {@link ByteBufferOutputStream}.
+	 * 
+	 * Like writers, readers also call {@link #newRead()} to get the current
+	 * {@link ByteBufferOutputStream} to read and must call
+	 * {@link #readCompleted()} after the end of read process.
+	 * 
+	 * @author sumanan
+	 * 
+	 */
 	public class ByteBufferArrayOutputStream extends OutputStream {
 
 		private final int debugPrint;
 
+		/**
+		 * Read index of {@link #bytebufferArray}.
+		 */
 		private int readIndex;
+
+		/**
+		 * Write index of {@link #bytebufferArray}.
+		 */
 		private int writeIndex;
+
 		private final ByteBufferOutputStream[] bytebufferArray;
+
+		/**
+		 * Keeps the {@link Status} of each element in the
+		 * {@link #bytebufferArray}
+		 */
 		private Map<Integer, AtomicReference<Status>> bufferStatus;
 
 		public ByteBufferArrayOutputStream(int listSize) {
@@ -521,6 +576,13 @@ public void write(byte b[], int off, int len) throws IOException {
 			bytebufferArray[writeIndex].write(b, off, len);
 		}
 
+		/**
+		 * Do not forget to call {@link #writeCompleted()} after every
+		 * successful bulk writes. Whatever written in between these two calls
+		 * will be captured into single {@link ByteBufferOutputStream}.
+		 * 
+		 * @return <code>true</code> iff the next buffer is free to write.
+		 */
 		public boolean newWrite() {
 			if (bufferStatus.get(writeIndex).compareAndSet(Status.canWrite,
 					Status.beingWritten)) {
@@ -538,6 +600,10 @@ public boolean newWrite() {
 			}
 		}
 
+		/**
+		 * Writer must call this method right after the writing of an collection
+		 * of objects is completed.
+		 */
 		public void writeCompleted() {
 			if (debugPrint > 0)
 				System.out.println(Thread.currentThread().getName()
@@ -551,6 +617,14 @@ public void writeCompleted() {
 				throw new IllegalStateException("bufferStatus conflict");
 		}
 
+		/**
+		 * Do not forget to call {@link #readCompleted()} after every successful
+		 * read of a {@link ByteBufferOutputStream}.
+		 * 
+		 * @return Next available {@link ByteBufferOutputStream} if available or
+		 *         <code>null</code> if no {@link ByteBufferOutputStream} is
+		 *         available to read.
+		 */
 		public synchronized ByteBufferOutputStream newRead() {
 			if (bufferStatus.get(readIndex).get() == Status.beingRead) {
 				if (debugPrint > 0)
@@ -579,6 +653,10 @@ public synchronized ByteBufferOutputStream newRead() {
 			}
 		}
 
+		/**
+		 * Reader must call this method right after the reading process is
+		 * completed.
+		 */
 		public void readCompleted() {
 			if (debugPrint > 0)
 				System.out.println(Thread.currentThread().getName()

From 7dcf5936431491365b3f35abad27216a458473d7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 09:19:40 +0800
Subject: [PATCH 234/881] Comments modified.

---
 .../mit/streamjit/impl/distributed/common/Connection.java   | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 26a1bf0f..806236eb 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -4,12 +4,8 @@
 import java.io.ObjectInputStream;
 import java.io.Serializable;
 
-import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.runtimer.Controller;
-
 /**
- * Communication interface for both {@link StreamNode} and {@link Controller}
- * side. This interface is for an IO connection that is already created, i.e.,
+ * Communication interface for an IO connection that is already created, i.e.,
  * creating a connections is not handled at here. Consider
  * {@link ConnectionFactory} to create a connection. </p> For the moment,
  * communicates at object granularity level. We may need to add primitive

From 156e6197e7ef34e830e1f2529ea0c36be8a355c7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 09:37:56 +0800
Subject: [PATCH 235/881] Bug in closeConnection() fixed.

Sets isConnected variable at the beginning of the method.
---
 src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java    |  3 +--
 .../distributed/common/AsynchronousTCPConnection.java  | 10 ++--------
 .../impl/distributed/common/TCPConnection.java         |  2 +-
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java b/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
index 763d1aa7..99afc30e 100644
--- a/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
+++ b/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
@@ -1,7 +1,6 @@
 package edu.mit.streamjit.impl.blob;
 
 import java.io.IOException;
-import java.util.concurrent.ExecutionException;
 
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
 
@@ -27,7 +26,7 @@ public boolean write(Object t) {
 	public int write(Object[] data, int offset, int length) {
 		try {
 			return con.write(data, offset, length);
-		} catch (IOException | InterruptedException | ExecutionException e) {
+		} catch (IOException e) {
 			e.printStackTrace();
 		}
 		return 0;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 31a00167..b83fda07 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -11,7 +11,6 @@
 import java.nio.channels.CompletionHandler;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
@@ -79,8 +78,7 @@ public void writeObject(Object obj) throws IOException {
 		 */
 	}
 
-	public int write(Object[] data, int offset, int length) throws IOException,
-			InterruptedException, ExecutionException {
+	public int write(Object[] data, int offset, int length) throws IOException {
 
 		final ObjectOutputStream objOS = this.ooStream;
 		final ByteBufferArrayOutputStream bBAos = this.bBAos;
@@ -133,13 +131,13 @@ public void failed(Throwable exc, ByteBuffer attachment) {
 	}
 
 	public final void closeConnection() {
+		isconnected = false;
 		try {
 			if (ooStream != null)
 				this.ooStream.close();
 			if (asyncSktChannel != null)
 				this.asyncSktChannel.close();
 		} catch (IOException ex) {
-			isconnected = false;
 			ex.printStackTrace();
 		}
 	}
@@ -156,10 +154,6 @@ public <T> T readObject() throws IOException, ClassNotFoundException {
 				"Reading object is not supported in asynchronous tcp mode");
 	}
 
-	public InetAddress getInetAddress() {
-		throw new java.lang.Error("Method not Implemented");
-	}
-
 	@Override
 	public void softClose() throws IOException {
 		while (!bBAos.newWrite());
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 9e9dd262..516883ed 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -93,6 +93,7 @@ public void writeObject(Object obj) throws IOException {
 	}
 
 	public final void closeConnection() {
+		isconnected = false;
 		try {
 			if (ooStream != null)
 				this.ooStream.close();
@@ -101,7 +102,6 @@ public final void closeConnection() {
 			if (socket != null)
 				this.socket.close();
 		} catch (IOException ex) {
-			isconnected = false;
 			ex.printStackTrace();
 		}
 	}

From 9815b6b679679cd1898bd9443d12eeb1a3c2831d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 10:44:18 +0800
Subject: [PATCH 236/881] Interface methods cleaned

Unnecessary interface methods isStillConnected(), closeConnection() and
getOtherNodeID() are removed from BoundaryChannel.
---
 .../distributed/common/BoundaryChannel.java     | 17 -----------------
 .../distributed/node/AsyncTCPOutputChannel.java | 16 ----------------
 .../impl/distributed/node/TCPInputChannel.java  | 13 +------------
 .../impl/distributed/node/TCPOutputChannel.java | 14 +-------------
 4 files changed, 2 insertions(+), 58 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
index 6f1efbc5..483827ce 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
@@ -20,29 +20,12 @@ public interface BoundaryChannel {
 
 	String name();
 
-	/**
-	 * Close the connection.
-	 * 
-	 * @throws IOException
-	 */
-	void closeConnection() throws IOException;
-
-	/**
-	 * @return true iff the connection with the other node is still valid.
-	 */
-	boolean isStillConnected();
-
 	/**
 	 * @return {@link Runnable} that does all IO communication and send
 	 *         data(stream tuples) to other node (or receive from other node).
 	 */
 	Runnable getRunnable();
 
-	/**
-	 * @return Other end of the node's ID.
-	 */
-	int getOtherNodeID();
-
 	public ImmutableList<Object> getUnprocessedData();
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 89f273db..09f6596f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -38,16 +38,6 @@ public String name() {
 		return name;
 	}
 
-	@Override
-	public final void closeConnection() throws IOException {
-		con.softClose();
-	}
-
-	@Override
-	public boolean isStillConnected() {
-		return (con == null) ? false : con.isStillConnected();
-	}
-
 	@Override
 	public Runnable getRunnable() {
 		return new Runnable() {
@@ -64,14 +54,8 @@ public void run() {
 		};
 	}
 
-	@Override
-	public int getOtherNodeID() {
-		return 0;
-	}
-
 	@Override
 	public ImmutableList<Object> getUnprocessedData() {
-		// TODO Auto-generated method stub
 		return null;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index 96da4752..ba6a8764 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -87,17 +87,11 @@ public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		writer = w;
 	}
 
-	@Override
-	public void closeConnection() throws IOException {
+	private void closeConnection() throws IOException {
 		// tcpConnection.closeConnection();
 		this.isClosed = true;
 	}
 
-	@Override
-	public boolean isStillConnected() {
-		return tcpConnection.isStillConnected();
-	}
-
 	@Override
 	public Runnable getRunnable() {
 		return new Runnable() {
@@ -339,11 +333,6 @@ private void reConnect() {
 		}
 	}
 
-	@Override
-	public int getOtherNodeID() {
-		return 0;
-	}
-
 	@Override
 	public void stop(int type) {
 		assert 0 < type && type < 4 : "Undefined stop type";
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index 55530942..aa9a6140 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -74,18 +74,11 @@ public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 		writer = w;
 	}
 
-	@Override
-	public final void closeConnection() throws IOException {
+	private void closeConnection() throws IOException {
 		// tcpConnection.closeConnection();
 		tcpConnection.softClose();
 	}
 
-	@Override
-	public final boolean isStillConnected() {
-		return (tcpConnection == null) ? false : tcpConnection
-				.isStillConnected();
-	}
-
 	@Override
 	public final Runnable getRunnable() {
 		return new Runnable() {
@@ -158,11 +151,6 @@ public final void sendData() {
 		}
 	}
 
-	@Override
-	public final int getOtherNodeID() {
-		return 0;
-	}
-
 	@Override
 	public final void stop(boolean isFinal) {
 		if (debugLevel > 0)

From 6a90b8eb78648668a4ac656aae58ee35442ff031 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 10:46:08 +0800
Subject: [PATCH 237/881] unnecessary imports removed

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 040c236d..f240bfb3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -5,9 +5,6 @@
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;

From fa3d3062835d06d79a1026d00bdb1d39968b4549 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 13:25:46 +0800
Subject: [PATCH 238/881] Access modifier removed from interface method

public ImmutableList<Object> getUnprocessedData() ->
ImmutableList<Object> getUnprocessedData();
---
 .../streamjit/impl/distributed/common/BoundaryChannel.java    | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
index 483827ce..1ec00d28 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
@@ -1,7 +1,5 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import java.io.IOException;
-
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.blob.Buffer;
@@ -26,7 +24,7 @@ public interface BoundaryChannel {
 	 */
 	Runnable getRunnable();
 
-	public ImmutableList<Object> getUnprocessedData();
+	ImmutableList<Object> getUnprocessedData();
 
 	/**
 	 * Interface that represents input channels.

From 1646e18d49b8ecca6fc55a53e5a1e9573c31811d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 16:36:54 +0800
Subject: [PATCH 239/881] Instance variales made final.

---
 src/edu/mit/streamjit/impl/distributed/common/Connection.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 806236eb..37715d00 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -80,9 +80,9 @@ public class ConnectionInfo implements Serializable {
 
 		private static final long serialVersionUID = 1L;
 
-		private int srcID;
+		private final int srcID;
 
-		private int dstID;
+		private final int dstID;
 
 		public ConnectionInfo(int srcID, int dstID) {
 			this.srcID = srcID;

From fbc7034e3afb8ad576826db3a130a8159c2727ce Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 16:57:17 +0800
Subject: [PATCH 240/881] isSymmetric flag added to ConnectionInfo

A connection info can be symmetric or asymmetric.
---
 .../impl/distributed/common/Connection.java   | 68 ++++++++++++++-----
 1 file changed, 50 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 37715d00..415fe455 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -65,10 +65,16 @@ public interface Connection {
 	public boolean isStillConnected();
 
 	/**
-	 * Describes a connection between two machines. ConnectionInfo is considered
+	 * Describes a connection between two machines.
+	 * <ol>
+	 * <li>if isSymmetric is <code>true</code>, ConnectionInfo is considered
 	 * symmetric for equal() and hashCode() calculation. As long as same
 	 * machineIDs are involved, irrespect of srcID and dstID positions, these
 	 * methods return same result.
+	 * <li>
+	 * if isSymmetric is <code>false</code> srcID and dstID will be treated as
+	 * not interchangeable entities.
+	 * </ol>
 	 * 
 	 * <p>
 	 * <b>Note : </b> All instances of ConnectionInfo, including subclass
@@ -84,9 +90,19 @@ public class ConnectionInfo implements Serializable {
 
 		private final int dstID;
 
+		/**
+		 * Tells whether this connection is symmetric or not.
+		 */
+		private final boolean isSymmetric;
+
 		public ConnectionInfo(int srcID, int dstID) {
+			this(srcID, dstID, true);
+		}
+
+		public ConnectionInfo(int srcID, int dstID, boolean isSymmetric) {
 			this.srcID = srcID;
 			this.dstID = dstID;
+			this.isSymmetric = isSymmetric;
 		}
 
 		public int getSrcID() {
@@ -97,17 +113,24 @@ public int getDstID() {
 			return dstID;
 		}
 
+		public boolean isSymmetric() {
+			return isSymmetric;
+		}
+
 		@Override
 		public int hashCode() {
 			final int prime = 31;
 			int result = 1;
-			/*
-			 * int min = Math.min(srcID, dstID); int max = Math.max(srcID,
-			 * dstID); result = prime * result + min; result = prime * result +
-			 * max;
-			 */
-			result = prime * result + srcID;
-			result = prime * result + dstID;
+			if (isSymmetric) {
+				int min = Math.min(srcID, dstID);
+				int max = Math.max(srcID, dstID);
+				result = prime * result + min;
+				result = prime * result + max;
+			} else {
+				result = prime * result + srcID;
+				result = prime * result + dstID;
+			}
+			result = prime * result + (isSymmetric ? 1231 : 1237);
 			return result;
 		}
 
@@ -120,22 +143,31 @@ public boolean equals(Object obj) {
 			if (!(obj instanceof ConnectionInfo))
 				return false;
 			ConnectionInfo other = (ConnectionInfo) obj;
-			/*
-			 * int myMin = Math.min(srcID, dstID); int myMax = Math.max(srcID,
-			 * dstID); int otherMin = Math.min(other.srcID, other.dstID); int
-			 * otherMax = Math.max(other.srcID, other.dstID); if (myMin !=
-			 * otherMin) return false; if (myMax != otherMax) return false;
-			 */
-			if (srcID != other.srcID)
-				return false;
-			if (dstID != other.dstID)
+			if (isSymmetric) {
+				int myMin = Math.min(srcID, dstID);
+				int myMax = Math.max(srcID, dstID);
+				int otherMin = Math.min(other.srcID, other.dstID);
+				int otherMax = Math.max(other.srcID, other.dstID);
+				if (myMin != otherMin)
+					return false;
+				if (myMax != otherMax)
+					return false;
+			} else {
+				if (srcID != other.srcID)
+					return false;
+				if (dstID != other.dstID)
+					return false;
+			}
+			if (isSymmetric != other.isSymmetric)
 				return false;
 			return true;
 		}
 
 		@Override
 		public String toString() {
-			return "ConnectionInfo [srcID=" + srcID + ", dstID=" + dstID + "]";
+			return String.format(
+					"ConnectionInfo [srcID=%d, dstID=%d, isSymmetric=%s]",
+					srcID, dstID, isSymmetric);
 		}
 	}
 }

From 98d9d94f3aabcbbae6e4dbcfcd59d8d8e1a40528 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 17:04:30 +0800
Subject: [PATCH 241/881] ConnectionInfo.makeConnection() added

ConnectionInfo will make connection based on its connection information.
---
 .../streamjit/impl/distributed/common/Connection.java | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 415fe455..cbda0b4c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -82,7 +82,7 @@ public interface Connection {
 	 * hashCode() and equals() methods. <b>The whole point of this class is to
 	 * identify a connection between two machines.</b>
 	 */
-	public class ConnectionInfo implements Serializable {
+	public abstract class ConnectionInfo implements Serializable {
 
 		private static final long serialVersionUID = 1L;
 
@@ -169,5 +169,14 @@ public String toString() {
 					"ConnectionInfo [srcID=%d, dstID=%d, isSymmetric=%s]",
 					srcID, dstID, isSymmetric);
 		}
+
+		/**
+		 * This function will establish a new connection according to the
+		 * connection info.
+		 * 
+		 * @return {@link Connection} that is described by this
+		 *         {@link ConnectionInfo}.
+		 */
+		public abstract Connection makeConnection(int myNodeID);
 	}
 }

From f75861839ff112170f5e8e9118ba32e5bef9e4ad Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 17:37:27 +0800
Subject: [PATCH 242/881] instanse variables from private to protected.

---
 .../impl/distributed/common/Connection.java         | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index cbda0b4c..81250c09 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -4,6 +4,8 @@
 import java.io.ObjectInputStream;
 import java.io.Serializable;
 
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
+
 /**
  * Communication interface for an IO connection that is already created, i.e.,
  * creating a connections is not handled at here. Consider
@@ -86,14 +88,14 @@ public abstract class ConnectionInfo implements Serializable {
 
 		private static final long serialVersionUID = 1L;
 
-		private final int srcID;
+		protected final int srcID;
 
-		private final int dstID;
+		protected final int dstID;
 
 		/**
 		 * Tells whether this connection is symmetric or not.
 		 */
-		private final boolean isSymmetric;
+		protected final boolean isSymmetric;
 
 		public ConnectionInfo(int srcID, int dstID) {
 			this(srcID, dstID, true);
@@ -174,9 +176,12 @@ public String toString() {
 		 * This function will establish a new connection according to the
 		 * connection info.
 		 * 
+		 * @param nodeID
+		 *            : nodeID of the {@link StreamNode} that invokes this
+		 *            method.
 		 * @return {@link Connection} that is described by this
 		 *         {@link ConnectionInfo}.
 		 */
-		public abstract Connection makeConnection(int myNodeID);
+		public abstract Connection makeConnection(int nodeID);
 	}
 }

From ef3cf2d652da6061aa0ca8b2ff3d83404fe99c3c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 17:53:09 +0800
Subject: [PATCH 243/881] NetworkInfo class has been added

This class will keeps all network related information that are needed to
establish connection between nodes.
---
 .../impl/distributed/common/Connection.java   |  5 +++-
 .../impl/distributed/common/NetworkInfo.java  | 28 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/NetworkInfo.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 81250c09..86600ecd 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -179,9 +179,12 @@ public String toString() {
 		 * @param nodeID
 		 *            : nodeID of the {@link StreamNode} that invokes this
 		 *            method.
+		 * @param networkInfo
+		 *            : network info of the system.
 		 * @return {@link Connection} that is described by this
 		 *         {@link ConnectionInfo}.
 		 */
-		public abstract Connection makeConnection(int nodeID);
+		public abstract Connection makeConnection(int nodeID,
+				NetworkInfo networkInfo, int timeOut);
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/NetworkInfo.java b/src/edu/mit/streamjit/impl/distributed/common/NetworkInfo.java
new file mode 100644
index 00000000..a7456228
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/NetworkInfo.java
@@ -0,0 +1,28 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import java.net.InetAddress;
+import java.util.Map;
+
+/**
+ * Keeps network information of all nodes in the system.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 23, 2014
+ */
+public class NetworkInfo {
+
+	private final Map<Integer, InetAddress> iNetAddressMap;
+
+	public NetworkInfo(Map<Integer, InetAddress> iNetAddressMap) {
+		this.iNetAddressMap = iNetAddressMap;
+	}
+
+	public InetAddress getInetAddress(int nodeID) {
+		if (this.iNetAddressMap == null)
+			return null;
+		InetAddress ipAddress = iNetAddressMap.get(nodeID);
+		if (ipAddress.isLoopbackAddress())
+			ipAddress = iNetAddressMap.get(0);
+		return ipAddress;
+	}
+}

From 00255a40c733974254a06d366eebd3a5319d1072 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 18:43:47 +0800
Subject: [PATCH 244/881] abstract class to concrete class

ConnectionInfo is made to a normal class from abstract class.
---
 .../mit/streamjit/impl/distributed/common/Connection.java | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 86600ecd..e71e3641 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -84,7 +84,7 @@ public interface Connection {
 	 * hashCode() and equals() methods. <b>The whole point of this class is to
 	 * identify a connection between two machines.</b>
 	 */
-	public abstract class ConnectionInfo implements Serializable {
+	public class ConnectionInfo implements Serializable {
 
 		private static final long serialVersionUID = 1L;
 
@@ -184,7 +184,9 @@ public String toString() {
 		 * @return {@link Connection} that is described by this
 		 *         {@link ConnectionInfo}.
 		 */
-		public abstract Connection makeConnection(int nodeID,
-				NetworkInfo networkInfo, int timeOut);
+		public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
+				int timeOut) {
+			throw new java.lang.Error("This method is not supposed to call");
+		}
 	}
 }

From 2108a9b2b2915c096e778cb46fb7d343b5c4d7c9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 18:53:13 +0800
Subject: [PATCH 245/881] TCPConnetionInfo.makeConnetion() implemented.

---
 .../distributed/common/TCPConnection.java     | 66 +++++++++++--------
 1 file changed, 40 insertions(+), 26 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 516883ed..966475bf 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -175,7 +175,7 @@ public static class TCPConnectionInfo extends ConnectionInfo {
 
 		private static final long serialVersionUID = 1L;
 
-		int portNo;
+		private final int portNo;
 
 		public TCPConnectionInfo(int srcID, int dstID, int portNo) {
 			super(srcID, dstID);
@@ -216,6 +216,34 @@ public String toString() {
 			return "TCPConnectionInfo [srcID=" + getSrcID() + ", dstID="
 					+ getDstID() + ", portID=" + portNo + "]";
 		}
+
+		@Override
+		public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
+				int timeOut) {
+			Connection con = null;
+			if (srcID == nodeID) {
+				try {
+					con = ConnectionFactory.getConnection(portNo, timeOut,
+							false);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+
+			else if (dstID == nodeID) {
+				InetAddress ipAddress = networkInfo.getInetAddress(srcID);
+				try {
+					con = ConnectionFactory.getConnection(
+							ipAddress.getHostAddress(), portNo, false);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			} else {
+				throw new IllegalArgumentException(
+						"Neither srcID nor dstID matches with nodeID");
+			}
+			return con;
+		}
 	}
 
 	/**
@@ -231,17 +259,16 @@ public String toString() {
 	 */
 	public static class TCPConnectionProvider {
 
-		private ConcurrentMap<TCPConnectionInfo, Connection> allConnections;
+		private ConcurrentMap<ConnectionInfo, Connection> allConnections;
 
 		private final int myNodeID;
 
-		private final Map<Integer, InetAddress> iNetAddressMap;
+		private final NetworkInfo networkInfo;
 
-		public TCPConnectionProvider(int myNodeID,
-				Map<Integer, InetAddress> iNetAddressMap) {
-			checkNotNull(iNetAddressMap, "nodeInfoMap is null");
+		public TCPConnectionProvider(int myNodeID, NetworkInfo networkInfo) {
+			checkNotNull(networkInfo, "networkInfo is null");
 			this.myNodeID = myNodeID;
-			this.iNetAddressMap = iNetAddressMap;
+			this.networkInfo = networkInfo;
 			this.allConnections = new ConcurrentHashMap<>();
 		}
 
@@ -252,7 +279,7 @@ public TCPConnectionProvider(int myNodeID,
 		 * @return
 		 * @throws IOException
 		 */
-		public Connection getConnection(TCPConnectionInfo conInfo)
+		public Connection getConnection(ConnectionInfo conInfo)
 				throws IOException {
 			return getConnection(conInfo, 0);
 		}
@@ -268,7 +295,7 @@ public Connection getConnection(TCPConnectionInfo conInfo)
 		 * @throws SocketTimeoutException
 		 * @throws IOException
 		 */
-		public Connection getConnection(TCPConnectionInfo conInfo, int timeOut)
+		public Connection getConnection(ConnectionInfo conInfo, int timeOut)
 				throws SocketTimeoutException, IOException {
 			Connection con = allConnections.get(conInfo);
 			if (con != null) {
@@ -280,22 +307,10 @@ public Connection getConnection(TCPConnectionInfo conInfo, int timeOut)
 				}
 			}
 
-			if (conInfo.getSrcID() == myNodeID) {
-				if (myNodeID == 0)
-					con = ConnectionFactory.getConnection(conInfo.getPortNo(),
-							timeOut, false);
-				else
-					con = ConnectionFactory.getAsyncConnection(conInfo
-							.getPortNo());
-			} else if (conInfo.getDstID() == myNodeID) {
-				InetAddress ipAddress = iNetAddressMap.get(conInfo.getSrcID());
-				if (ipAddress.isLoopbackAddress())
-					ipAddress = iNetAddressMap.get(0);
-
-				int portNo = conInfo.getPortNo();
-				con = ConnectionFactory.getConnection(
-						ipAddress.getHostAddress(), portNo, false);
-			}
+			con = conInfo.makeConnection(myNodeID, networkInfo, timeOut);
+			if (con == null)
+				throw new IOException("Connection making process failed.");
+
 			allConnections.put(conInfo, con);
 			return con;
 		}
@@ -305,7 +320,6 @@ public void closeAllConnections() {
 				try {
 					con.closeConnection();
 				} catch (IOException e) {
-					// TODO Auto-generated catch block
 					e.printStackTrace();
 				}
 			}

From b8c3cdc205432db7120393d0866757c9fbf2c399 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 22 May 2014 18:56:15 +0800
Subject: [PATCH 246/881] AsyncTCPConnectionInfo has been added

---
 .../common/AsynchronousTCPConnection.java     | 81 ++++++++++++++++++-
 1 file changed, 79 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index b83fda07..22b1461c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -5,7 +5,6 @@
 import java.io.ObjectOutputStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
-import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousSocketChannel;
 import java.nio.channels.CompletionHandler;
@@ -13,6 +12,8 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
+
 /**
  * Uses {@link AsynchronousSocketChannel} from Java's NIO.2 to send data. This
  * class only supports bulk asynchronous write. Reads ({@link #readObject()}) or
@@ -156,7 +157,8 @@ public <T> T readObject() throws IOException, ClassNotFoundException {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!bBAos.newWrite());
+		while (!bBAos.newWrite())
+			;
 		this.ooStream.write('\u001a');
 		this.ooStream.flush();
 		bBAos.writeCompleted();
@@ -665,4 +667,79 @@ public void readCompleted() {
 				throw new IllegalStateException("bufferStatus conflict");
 		}
 	}
+
+	/**
+	 * Uniquely identifies a Asynchronous TCP connection among all connected
+	 * machines.
+	 * 
+	 * <p>
+	 * NOTE: IPAddress is not included for the moment to avoid re-sending same
+	 * information again and again for every reconfiguration. machineId to
+	 * {@link NodeInfo} map will be sent initially. So {@link StreamNode}s can
+	 * get ipAddress of a machine from that map.
+	 */
+	public static class AsyncTCPConnectionInfo extends ConnectionInfo {
+
+		private static final long serialVersionUID = 1L;
+
+		private final int portNo;
+
+		public AsyncTCPConnectionInfo(int srcID, int dstID, int portNo) {
+			super(srcID, dstID, false);
+			Ipv4Validator validator = Ipv4Validator.getInstance();
+			if (!validator.isValid(portNo))
+				throw new IllegalArgumentException("Invalid port No");
+			this.portNo = portNo;
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = super.hashCode();
+			result = prime * result + portNo;
+			return result;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj)
+				return true;
+			if (!super.equals(obj))
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			AsyncTCPConnectionInfo other = (AsyncTCPConnectionInfo) obj;
+			if (portNo != other.portNo)
+				return false;
+			return true;
+		}
+
+		@Override
+		public String toString() {
+			return "AsyncTCPConnectionInfo [srcID=" + getSrcID() + ", dstID="
+					+ getDstID() + ", portID=" + portNo + "]";
+		}
+
+		@Override
+		public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
+				int timeOut) {
+			Connection con = null;
+			if (srcID == nodeID) {
+				try {
+					con = ConnectionFactory.getAsyncConnection(portNo);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+
+			else if (dstID == nodeID) {
+				throw new IllegalStateException(
+						"Only senders can use AsynchronousTCPConnection");
+			} else {
+				throw new IllegalArgumentException(
+						"Neither srcID nor dstID matches with nodeID");
+			}
+			return con;
+		}
+	}
 }

From 730e003fcfe73a804f9b7416065b7f961757c9fe Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 23 May 2014 09:05:51 +0800
Subject: [PATCH 247/881] TCPConnectionInfo -> ConnectionInfo

Classes refer super type.
---
 .../impl/distributed/HeadChannel.java         |  4 +-
 .../impl/distributed/StreamJitAppManager.java | 16 +++----
 .../impl/distributed/TailChannel.java         |  4 +-
 .../distributed/common/MiscCtrlElements.java  |  6 +--
 .../impl/distributed/common/SNException.java  |  6 +--
 .../node/AsyncTCPOutputChannel.java           |  6 +--
 .../distributed/node/BlobsManagerImpl.java    | 10 ++--
 .../node/CfgStringProcessorImpl.java          | 11 +++--
 .../distributed/node/TCPInputChannel.java     |  6 +--
 .../distributed/node/TCPOutputChannel.java    |  6 +--
 .../impl/distributed/runtimer/Controller.java | 47 ++++++++++++-------
 11 files changed, 69 insertions(+), 53 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 698f029b..9844428e 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -5,7 +5,7 @@
 import edu.mit.streamjit.impl.blob.AbstractReadOnlyBuffer;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
 
@@ -19,7 +19,7 @@
 public class HeadChannel extends TCPOutputChannel {
 
 	public HeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 2fdfe412..ecf0f923 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -23,6 +23,7 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Command;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
@@ -36,7 +37,6 @@
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 public class StreamJitAppManager {
@@ -83,7 +83,7 @@ public class StreamJitAppManager {
 
 	private volatile AppStatus status;
 
-	private Map<Token, TCPConnectionInfo> conInfoMap;
+	private Map<Token, ConnectionInfo> conInfoMap;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
 			ConfigurationManager cfgManager) {
@@ -164,10 +164,10 @@ public boolean reconfigure(int multiplier) {
 	 * @param cfg
 	 * @param bufferMap
 	 */
-	private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
+	private void setupHeadTail(Map<Token, ConnectionInfo> conInfoMap,
 			ImmutableMap<Token, Buffer> bufferMap, int multiplier) {
 
-		TCPConnectionInfo headconInfo = conInfoMap.get(headToken);
+		ConnectionInfo headconInfo = conInfoMap.get(headToken);
 		assert headconInfo != null : "No head connection info exists in conInfoMap";
 		assert headconInfo.getSrcID() == controller.controllerNodeID
 				|| headconInfo.getDstID() == controller.controllerNodeID : "Head channel should start from the controller. "
@@ -181,7 +181,7 @@ private void setupHeadTail(Map<Token, TCPConnectionInfo> conInfoMap,
 				controller.getConProvider(), headconInfo, "headChannel - "
 						+ headToken.toString(), 0);
 
-		TCPConnectionInfo tailconInfo = conInfoMap.get(tailToken);
+		ConnectionInfo tailconInfo = conInfoMap.get(tailToken);
 		assert tailconInfo != null : "No tail connection info exists in conInfoMap";
 		assert tailconInfo.getSrcID() == controller.controllerNodeID
 				|| tailconInfo.getDstID() == controller.controllerNodeID : "Tail channel should ends at the controller. "
@@ -368,7 +368,7 @@ private class SNExceptionProcessorImpl implements SNExceptionProcessor {
 
 		private final Object abExLock = new Object();
 
-		private Set<TCPConnectionInfo> exConInfos;
+		private Set<ConnectionInfo> exConInfos;
 
 		private SNExceptionProcessorImpl() {
 			exConInfos = new HashSet<>();
@@ -388,7 +388,7 @@ public void process(AddressBindException abEx) {
 				}
 
 				Token t = null;
-				for (Map.Entry<Token, TCPConnectionInfo> entry : conInfoMap
+				for (Map.Entry<Token, ConnectionInfo> entry : conInfoMap
 						.entrySet()) {
 					if (abEx.conInfo.equals(entry.getValue())) {
 						t = entry.getKey();
@@ -401,7 +401,7 @@ public void process(AddressBindException abEx) {
 							"Illegal TCP connection - " + abEx.conInfo);
 				}
 
-				TCPConnectionInfo coninfo = controller
+				ConnectionInfo coninfo = controller
 						.getNewTCPConInfo(abEx.conInfo);
 
 				exConInfos.add(abEx.conInfo);
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index ebe13a85..0377ba87 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -10,8 +10,8 @@
 
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
 
@@ -48,7 +48,7 @@ public class TailChannel extends TCPInputChannel {
 	 *            of outputs at the beginning).
 	 */
 	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel,
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int skipCount, int steadyCount) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 		this.skipCount = skipCount;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/MiscCtrlElements.java b/src/edu/mit/streamjit/impl/distributed/common/MiscCtrlElements.java
index d7e9a2dc..3877c0ea 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/MiscCtrlElements.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/MiscCtrlElements.java
@@ -1,7 +1,7 @@
 package edu.mit.streamjit.impl.distributed.common;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
 public abstract class MiscCtrlElements implements CTRLRMessageElement {
 
@@ -17,10 +17,10 @@ public void accept(CTRLRMessageVisitor visitor) {
 	public static final class NewConInfo extends MiscCtrlElements {
 		private static final long serialVersionUID = 1L;
 
-		public final TCPConnectionInfo conInfo;
+		public final ConnectionInfo conInfo;
 		public final Token token;
 
-		public NewConInfo(TCPConnectionInfo conInfo, Token token) {
+		public NewConInfo(ConnectionInfo conInfo, Token token) {
 			this.conInfo = conInfo;
 			this.token = token;
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNException.java b/src/edu/mit/streamjit/impl/distributed/common/SNException.java
index 1ff8421b..5c1e4ab5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNException.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNException.java
@@ -1,6 +1,6 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
 public class SNException implements SNMessageElement {
 
@@ -18,9 +18,9 @@ public void accept(SNMessageVisitor visitor) {
 	public static final class AddressBindException extends SNException {
 		private static final long serialVersionUID = 1L;
 
-		public final TCPConnectionInfo conInfo;
+		public final ConnectionInfo conInfo;
 
-		public AddressBindException(TCPConnectionInfo conInfo) {
+		public AddressBindException(ConnectionInfo conInfo) {
 			this.conInfo = conInfo;
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 09f6596f..98f63f0e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -7,7 +7,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
 import edu.mit.streamjit.impl.distributed.common.Connection;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 
 public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
@@ -18,14 +18,14 @@ public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
 	private final TCPConnectionProvider conProvider;
 
-	TCPConnectionInfo conInfo;
+	ConnectionInfo conInfo;
 
 	private volatile boolean isFinal;
 
 	private volatile boolean stopCalled;
 
 	public AsyncTCPOutputChannel(TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		name = "AsyncTCPOutputChannel " + bufferTokenName;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 41176ac0..81769fbe 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -32,11 +32,11 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
@@ -52,7 +52,7 @@ public class BlobsManagerImpl implements BlobsManager {
 	private Map<Token, BlobExecuter> blobExecuters;
 	private final StreamNode streamNode;
 	private final TCPConnectionProvider conProvider;
-	private Map<Token, TCPConnectionInfo> conInfoMap;
+	private Map<Token, ConnectionInfo> conInfoMap;
 
 	private Set<Token> globalOutputTokens;
 
@@ -74,7 +74,7 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final boolean useDrainDeadLockHandler;
 
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
-			Map<Token, TCPConnectionInfo> conInfoMap, StreamNode streamNode,
+			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
 			TCPConnectionProvider conProvider) {
 		this.conInfoMap = conInfoMap;
 		this.streamNode = streamNode;
@@ -231,7 +231,7 @@ private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 			Set<Token> inputTokens, ImmutableMap<Token, Buffer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryInputChannel> inputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : inputTokens) {
-			TCPConnectionInfo conInfo = conInfoMap.get(t);
+			ConnectionInfo conInfo = conInfoMap.get(t);
 			inputChannelMap.put(t, new TCPInputChannel(bufferMap.get(t),
 					conProvider, conInfo, t.toString(), 0));
 		}
@@ -242,7 +242,7 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 			Set<Token> outputTokens, ImmutableMap<Token, Buffer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryOutputChannel> outputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : outputTokens) {
-			TCPConnectionInfo conInfo = conInfoMap.get(t);
+			ConnectionInfo conInfo = conInfoMap.get(t);
 			outputChannelMap.put(t, new AsyncTCPOutputChannel(conProvider,
 					conInfo, t.toString(), 0));
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index bc3c94db..44976174 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -30,7 +30,8 @@
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
@@ -64,8 +65,10 @@ public void process(String json, ConfigType type, DrainData drainData) {
 				Map<Integer, InetAddress> iNetAddressMap = (Map<Integer, InetAddress>) staticConfig
 						.getExtraData(GlobalConstants.INETADDRESS_MAP);
 
+				NetworkInfo networkInfo = new NetworkInfo(iNetAddressMap);
+
 				this.conProvider = new TCPConnectionProvider(
-						streamNode.getNodeID(), iNetAddressMap);
+						streamNode.getNodeID(), networkInfo);
 			} else
 				System.err
 						.println("New static configuration received...But Ignored...");
@@ -90,7 +93,7 @@ public void process(String json, ConfigType type, DrainData drainData) {
 					e.printStackTrace();
 				}
 
-				Map<Token, TCPConnectionInfo> conInfoMap = (Map<Token, TCPConnectionInfo>) cfg
+				Map<Token, ConnectionInfo> conInfoMap = (Map<Token, ConnectionInfo>) cfg
 						.getExtraData(GlobalConstants.CONINFOMAP);
 
 				streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
@@ -216,7 +219,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 		URL url;
 		try {
 			url = jarFile.toURI().toURL();
-			URL[] urls = new URL[]{url};
+			URL[] urls = new URL[] { url };
 
 			ClassLoader loader = new URLClassLoader(urls);
 			Class<?> topStreamClass;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index ba6a8764..74bd237f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -15,7 +15,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 
 /**
@@ -44,7 +44,7 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private final TCPConnectionProvider conProvider;
 
-	private final TCPConnectionInfo conInfo;
+	private final ConnectionInfo conInfo;
 
 	private Connection tcpConnection;
 
@@ -61,7 +61,7 @@ public class TCPInputChannel implements BoundaryInputChannel {
 	private ImmutableList<Object> unProcessedData;
 
 	public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index aa9a6140..7c519ab9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -10,7 +10,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 
 /**
@@ -35,7 +35,7 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	private final TCPConnectionProvider conProvider;
 
-	private final TCPConnectionInfo conInfo;
+	private final ConnectionInfo conInfo;
 
 	private Connection tcpConnection;
 
@@ -50,7 +50,7 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 	protected ImmutableList<Object> unProcessedData;
 
 	public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			TCPConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
 		this.conInfo = conInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 84533dd1..d3ef0656 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -19,9 +19,11 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
+import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
@@ -58,7 +60,7 @@ public class Controller {
 	 */
 	public final int controllerNodeID;
 
-	private Set<TCPConnectionInfo> currentConInfos;
+	private Set<ConnectionInfo> currentConInfos;
 
 	public Controller() {
 		this.comManager = new BlockingCommunicationManager();
@@ -128,20 +130,22 @@ public void newApp(Configuration staticCfg) {
 
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
 		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
-		this.conProvider = new TCPConnectionProvider(controllerNodeID, inetMap);
+		NetworkInfo networkinfo = new NetworkInfo(inetMap);
+		this.conProvider = new TCPConnectionProvider(controllerNodeID,
+				networkinfo);
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
 		sendToAll(json);
 	}
 
-	public Map<Token, TCPConnectionInfo> buildConInfoMap(
+	public Map<Token, ConnectionInfo> buildConInfoMap(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap,
 			Worker<?, ?> source, Worker<?, ?> sink) {
 
 		assert partitionsMachineMap != null : "partitionsMachineMap is null";
 
-		Set<TCPConnectionInfo> usedConInfos = new HashSet<>();
-		Map<Token, TCPConnectionInfo> conInfoMap = new HashMap<>();
+		Set<ConnectionInfo> usedConInfos = new HashSet<>();
+		Map<Token, ConnectionInfo> conInfoMap = new HashMap<>();
 
 		for (Integer machineID : partitionsMachineMap.keySet()) {
 			List<Set<Worker<?, ?>>> blobList = partitionsMachineMap
@@ -187,15 +191,15 @@ public Map<Token, TCPConnectionInfo> buildConInfoMap(
 	 * {@link #buildConInfoMap(Map, Worker, Worker)} method.
 	 */
 	private void addtoconInfoMap(int srcID, int dstID, Token t,
-			Set<TCPConnectionInfo> usedConInfos,
-			Map<Token, TCPConnectionInfo> conInfoMap) {
+			Set<ConnectionInfo> usedConInfos,
+			Map<Token, ConnectionInfo> conInfoMap) {
 
 		ConnectionInfo conInfo = new ConnectionInfo(srcID, dstID);
 
-		List<TCPConnectionInfo> conSet = getTcpConInfo(conInfo);
-		TCPConnectionInfo tcpConInfo = null;
+		List<ConnectionInfo> conSet = getTcpConInfo(conInfo);
+		ConnectionInfo tcpConInfo = null;
 
-		for (TCPConnectionInfo con : conSet) {
+		for (ConnectionInfo con : conSet) {
 			if (!usedConInfos.contains(con)) {
 				tcpConInfo = con;
 				break;
@@ -203,7 +207,11 @@ private void addtoconInfoMap(int srcID, int dstID, Token t,
 		}
 
 		if (tcpConInfo == null) {
-			tcpConInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
+			if (srcID == 0)
+				tcpConInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
+			else
+				tcpConInfo = new AsyncTCPConnectionInfo(srcID, dstID,
+						startPortNo++);
 			this.currentConInfos.add(tcpConInfo);
 		}
 
@@ -211,9 +219,9 @@ private void addtoconInfoMap(int srcID, int dstID, Token t,
 		usedConInfos.add(tcpConInfo);
 	}
 
-	private List<TCPConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
-		List<TCPConnectionInfo> conList = new ArrayList<>();
-		for (TCPConnectionInfo tcpconInfo : currentConInfos) {
+	private List<ConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
+		List<ConnectionInfo> conList = new ArrayList<>();
+		for (ConnectionInfo tcpconInfo : currentConInfos) {
 			if (conInfo.equals(tcpconInfo))
 				conList.add(tcpconInfo);
 		}
@@ -281,11 +289,16 @@ public void registerManager(StreamJitAppManager manager) {
 		}
 	}
 
-	public TCPConnectionInfo getNewTCPConInfo(TCPConnectionInfo conInfo) {
+	public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
 		if (currentConInfos.contains(conInfo))
 			currentConInfos.remove(conInfo);
-		TCPConnectionInfo newConinfo = new TCPConnectionInfo(
-				conInfo.getSrcID(), conInfo.getDstID(), startPortNo++);
+		ConnectionInfo newConinfo;
+		if (conInfo.getSrcID() == 0)
+			newConinfo = new TCPConnectionInfo(conInfo.getSrcID(),
+					conInfo.getDstID(), startPortNo++);
+		else
+			newConinfo = new AsyncTCPConnectionInfo(conInfo.getSrcID(),
+					conInfo.getDstID(), startPortNo++);
 		currentConInfos.add(newConinfo);
 
 		return newConinfo;

From 4acf7474d7325684319cd44f397dfdf20e4982ee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 23 May 2014 09:06:58 +0800
Subject: [PATCH 248/881] Makes blocking connection if destination.

AsynchronousTCPConnectionInfo makes blocking TCP connection if the node
that tries to make connection is destination node.
---
 .../common/AsynchronousTCPConnection.java           | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 22b1461c..35439744 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -5,6 +5,7 @@
 import java.io.ObjectOutputStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
+import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.AsynchronousSocketChannel;
 import java.nio.channels.CompletionHandler;
@@ -157,8 +158,7 @@ public <T> T readObject() throws IOException, ClassNotFoundException {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!bBAos.newWrite())
-			;
+		while (!bBAos.newWrite());
 		this.ooStream.write('\u001a');
 		this.ooStream.flush();
 		bBAos.writeCompleted();
@@ -733,8 +733,13 @@ public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
 			}
 
 			else if (dstID == nodeID) {
-				throw new IllegalStateException(
-						"Only senders can use AsynchronousTCPConnection");
+				InetAddress ipAddress = networkInfo.getInetAddress(srcID);
+				try {
+					con = ConnectionFactory.getConnection(
+							ipAddress.getHostAddress(), portNo, false);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
 			} else {
 				throw new IllegalArgumentException(
 						"Neither srcID nor dstID matches with nodeID");

From 295a63da2fb7e81e6f9c058dc6c1738ae7830693 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 23 May 2014 13:40:32 +0800
Subject: [PATCH 249/881] Bug in the equals() method corrected

equals() breaks the reflexive(), symmetric() and transitive() properties,
especially when subclasses involves. The purpose of this overwriting is
to check whether an already established connection could be reused.
---
 .../impl/distributed/common/Connection.java       | 15 +++++++++++----
 .../impl/distributed/common/TCPConnection.java    |  2 +-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index e71e3641..1cd0e11a 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -101,7 +101,7 @@ public ConnectionInfo(int srcID, int dstID) {
 			this(srcID, dstID, true);
 		}
 
-		public ConnectionInfo(int srcID, int dstID, boolean isSymmetric) {
+		protected ConnectionInfo(int srcID, int dstID, boolean isSymmetric) {
 			this.srcID = srcID;
 			this.dstID = dstID;
 			this.isSymmetric = isSymmetric;
@@ -136,6 +136,15 @@ public int hashCode() {
 			return result;
 		}
 
+		/*
+		 * (non-Javadoc)
+		 * 
+		 * @see java.lang.Object#equals(java.lang.Object) equals() overwritten
+		 * here breaks the reflexive(), symmetric() and transitive() properties,
+		 * especially when subclasses involves. The purpose of this overwriting
+		 * is to check whether an already established connection could be
+		 * reused.
+		 */
 		@Override
 		public boolean equals(Object obj) {
 			if (this == obj)
@@ -145,7 +154,7 @@ public boolean equals(Object obj) {
 			if (!(obj instanceof ConnectionInfo))
 				return false;
 			ConnectionInfo other = (ConnectionInfo) obj;
-			if (isSymmetric) {
+			if (other.isSymmetric) {
 				int myMin = Math.min(srcID, dstID);
 				int myMax = Math.max(srcID, dstID);
 				int otherMin = Math.min(other.srcID, other.dstID);
@@ -160,8 +169,6 @@ public boolean equals(Object obj) {
 				if (dstID != other.dstID)
 					return false;
 			}
-			if (isSymmetric != other.isSymmetric)
-				return false;
 			return true;
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 966475bf..76843340 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -178,7 +178,7 @@ public static class TCPConnectionInfo extends ConnectionInfo {
 		private final int portNo;
 
 		public TCPConnectionInfo(int srcID, int dstID, int portNo) {
-			super(srcID, dstID);
+			super(srcID, dstID, true);
 			Ipv4Validator validator = Ipv4Validator.getInstance();
 			if (!validator.isValid(portNo))
 				throw new IllegalArgumentException("Invalid port No");

From 80d20b98727128be955d991b1a8fa81c15d3c2bb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 23 May 2014 13:43:17 +0800
Subject: [PATCH 250/881] Testing equals() and hashCode() implementations

---
 .../impl/distributed/common/Tester.java       | 111 +++++++++++++++++-
 1 file changed, 108 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Tester.java b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
index 199dee25..1319a911 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Tester.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
@@ -7,6 +7,12 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
 public class Tester {
 
@@ -14,7 +20,14 @@ public class Tester {
 	 * @param args
 	 */
 	public static void main(String[] args) {
+		// test1();
+		test2();
+	}
 
+	/**
+	 * Testing one - tests the size of an object.
+	 */
+	private static void test1() {
 		Error er = Error.FILE_NOT_FOUND;
 		AppStatus apSts = AppStatus.STOPPED;
 		ByteArrayOutputStream byteAos = new ByteArrayOutputStream();
@@ -33,10 +46,102 @@ public static void main(String[] args) {
 			e.printStackTrace();
 		}
 
-		/*
-		 * try { os.writeInt(34345); } catch (IOException e) { e.printStackTrace(); }
-		 */
+		try {
+			os.writeInt(34345);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
 
 		System.out.println(byteAos.toByteArray().length);
 	}
+
+	/**
+	 * Tests the equals and hascode.
+	 */
+	private static void test2() {
+
+		ConnectionInfo asyConInfo1 = new AsyncTCPConnectionInfo(1, 4, 8989);
+		ConnectionInfo asyConInfo2 = new AsyncTCPConnectionInfo(1, 4, 8980);
+		ConnectionInfo asyConInfo3 = new AsyncTCPConnectionInfo(4, 1, 8989);
+		ConnectionInfo asyConInfo4 = new AsyncTCPConnectionInfo(4, 1, 8980);
+		ConnectionInfo asyConInfo5 = new AsyncTCPConnectionInfo(1, 4, 8989);
+
+		ConnectionInfo tcpConInfo1 = new TCPConnectionInfo(1, 4, 8989);
+		ConnectionInfo tcpConInfo2 = new TCPConnectionInfo(1, 4, 8980);
+		ConnectionInfo tcpConInfo3 = new TCPConnectionInfo(4, 1, 8989);
+		ConnectionInfo tcpConInfo4 = new TCPConnectionInfo(4, 1, 8980);
+
+		ConnectionInfo conInfo1 = new ConnectionInfo(1, 4, true);
+		ConnectionInfo conInfo2 = new ConnectionInfo(1, 4, false);
+		ConnectionInfo conInfo3 = new ConnectionInfo(4, 1, true);
+		ConnectionInfo conInfo4 = new ConnectionInfo(4, 1, false);
+
+		System.out.println("AsyncTCPConnectionInfo - AsyncTCPConnectionInfo");
+		System.out.println(asyConInfo1.equals(asyConInfo2));
+		System.out.println(asyConInfo1.equals(asyConInfo3));
+		System.out.println(asyConInfo1.equals(asyConInfo4));
+		System.out.println(asyConInfo2.equals(asyConInfo3));
+		System.out.println(asyConInfo2.equals(asyConInfo4));
+		System.out.println(asyConInfo3.equals(asyConInfo4));
+		System.out.println();
+
+		System.out.println("ConnectionInfo - AsyncTCPConnectionInfo");
+		System.out.println(conInfo1.equals(asyConInfo1));
+		System.out.println(conInfo1.equals(asyConInfo2));
+		System.out.println(conInfo1.equals(asyConInfo3));
+		System.out.println(conInfo1.equals(asyConInfo4));
+		System.out.println(conInfo2.equals(asyConInfo1));
+		System.out.println(conInfo2.equals(asyConInfo2));
+		System.out.println(conInfo2.equals(asyConInfo3));
+		System.out.println(conInfo2.equals(asyConInfo4));
+		System.out.println(conInfo3.equals(asyConInfo1));
+		System.out.println(conInfo3.equals(asyConInfo2));
+		System.out.println(conInfo3.equals(asyConInfo3));
+		System.out.println(conInfo3.equals(asyConInfo4));
+		System.out.println(conInfo4.equals(asyConInfo1));
+		System.out.println(conInfo4.equals(asyConInfo2));
+		System.out.println(conInfo4.equals(asyConInfo3));
+		System.out.println(conInfo4.equals(asyConInfo4));
+		System.out.println();
+
+		System.out.println("ConnectionInfo - TCPConnectionInfo");
+		System.out.println(conInfo1.equals(tcpConInfo1));
+		System.out.println(conInfo1.equals(tcpConInfo2));
+		System.out.println(conInfo1.equals(tcpConInfo3));
+		System.out.println(conInfo1.equals(tcpConInfo4));
+		System.out.println(conInfo2.equals(tcpConInfo1));
+		System.out.println(conInfo2.equals(tcpConInfo2));
+		System.out.println(conInfo2.equals(tcpConInfo3));
+		System.out.println(conInfo2.equals(tcpConInfo4));
+		System.out.println(conInfo3.equals(tcpConInfo1));
+		System.out.println(conInfo3.equals(tcpConInfo2));
+		System.out.println(conInfo3.equals(tcpConInfo3));
+		System.out.println(conInfo3.equals(tcpConInfo4));
+		System.out.println(conInfo4.equals(tcpConInfo1));
+		System.out.println(conInfo4.equals(tcpConInfo2));
+		System.out.println(conInfo4.equals(tcpConInfo3));
+		System.out.println(conInfo4.equals(tcpConInfo4));
+		System.out.println();
+
+		Map<ConnectionInfo, Integer> tesMap = new HashMap<>();
+		tesMap.put(tcpConInfo1, 1);
+		tesMap.put(asyConInfo1, 2);
+
+		System.out.println(tesMap.containsKey(tcpConInfo1));
+		System.out.println(tesMap.containsKey(tcpConInfo2));
+		System.out.println(tesMap.containsKey(tcpConInfo3));
+		System.out.println(tesMap.containsKey(tcpConInfo4));
+
+		System.out.println(tesMap.containsKey(asyConInfo1));
+		System.out.println(tesMap.containsKey(asyConInfo2));
+		System.out.println(tesMap.containsKey(asyConInfo3));
+		System.out.println(tesMap.containsKey(asyConInfo4));
+		System.out.println(tesMap.containsKey(asyConInfo5));
+
+		System.out.println(tesMap.containsKey(conInfo1));
+		System.out.println(tesMap.containsKey(conInfo2));
+		System.out.println(tesMap.containsKey(conInfo3));
+		System.out.println(tesMap.containsKey(conInfo4));
+	}
+
 }

From abb06ea8e9d23b64d4e0c2bf39540fe915097f04 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 May 2014 09:13:46 +0800
Subject: [PATCH 251/881] 2 methods have been added to BoundaryChannel
 interface

Connection getConnection();
ConnectionInfo getConnectionInfo();
---
 .../impl/distributed/common/BoundaryChannel.java    |  5 +++++
 .../distributed/node/AsyncTCPOutputChannel.java     | 13 +++++++++----
 .../impl/distributed/node/BlobsManagerImpl.java     |  5 ++++-
 .../impl/distributed/node/TCPInputChannel.java      | 12 ++++++++++++
 .../impl/distributed/node/TCPOutputChannel.java     | 10 ++++++++++
 5 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
index 1ec00d28..1ebc4bf9 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
@@ -3,6 +3,7 @@
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
 /**
  * {@link BoundaryChannel} wraps a {@link Buffer} that crosses over the
@@ -26,6 +27,10 @@ public interface BoundaryChannel {
 
 	ImmutableList<Object> getUnprocessedData();
 
+	Connection getConnection();
+
+	ConnectionInfo getConnectionInfo();
+
 	/**
 	 * Interface that represents input channels.
 	 */
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 98f63f0e..960ed741 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -5,7 +5,6 @@
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
@@ -18,7 +17,7 @@ public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
 	private final TCPConnectionProvider conProvider;
 
-	ConnectionInfo conInfo;
+	private final ConnectionInfo conInfo;
 
 	private volatile boolean isFinal;
 
@@ -77,7 +76,13 @@ public void sendData() {
 
 	}
 
-	public AsynchronousTCPConnection getConnection() {
-		return (AsynchronousTCPConnection) con;
+	@Override
+	public Connection getConnection() {
+		return con;
+	}
+
+	@Override
+	public ConnectionInfo getConnectionInfo() {
+		return conInfo;
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 81769fbe..fc996bf0 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -24,6 +24,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
@@ -330,7 +331,9 @@ private void start() {
 				if (globalOutputTokens.contains(t)) {
 					AsyncTCPOutputChannel asyChannel = (AsyncTCPOutputChannel) outputChannels
 							.get(t);
-					Buffer b = new AsyncTCPBuffer(asyChannel.getConnection());
+					Buffer b = new AsyncTCPBuffer(
+							(AsynchronousTCPConnection) asyChannel
+									.getConnection());
 					bufferMapBuilder.put(t, b);
 				} else {
 					Buffer b = bufferMap.get(t);
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index 74bd237f..f8614481 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -434,4 +434,16 @@ public ImmutableList<Object> getUnprocessedData() {
 
 		return unProcessedData;
 	}
+
+	@Override
+	public Connection getConnection() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public ConnectionInfo getConnectionInfo() {
+		// TODO Auto-generated method stub
+		return null;
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index 7c519ab9..d1b0293d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -237,4 +237,14 @@ public ImmutableList<Object> getUnprocessedData() {
 					"Still processing... No unprocessed data");
 		return unProcessedData;
 	}
+
+	@Override
+	public Connection getConnection() {
+		return tcpConnection;
+	}
+
+	@Override
+	public ConnectionInfo getConnectionInfo() {
+		return conInfo;
+	}
 }

From a4d5fd3a8e76acf4f9da6010e15b09c14801b374 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 May 2014 08:16:05 +0800
Subject: [PATCH 252/881] Boundary channel factory has been added

This channel factory will be responsible to creates all kind of
BoundaryChannels.
---
 .../common/BoundaryChannelFactory.java        | 74 +++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
new file mode 100644
index 00000000..dd6d18ed
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -0,0 +1,74 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
+
+/**
+ * {@link BoundaryChannel} maker.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 28, 2014
+ */
+public interface BoundaryChannelFactory {
+
+	BoundaryInputChannel makeInputChannel(Token t, Buffer buffer,
+			ConnectionInfo conInfo);
+
+	BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
+			ConnectionInfo conInfo);
+
+	/**
+	 * Makes blocking {@link TCPInputChannel} and {@link TCPOutputChannel}.
+	 * 
+	 */
+	public static class TCPBoundaryChannelFactory
+			implements
+				BoundaryChannelFactory {
+
+		protected final TCPConnectionProvider conProvider;
+
+		public TCPBoundaryChannelFactory(TCPConnectionProvider conProvider) {
+			this.conProvider = conProvider;
+		}
+
+		@Override
+		public BoundaryInputChannel makeInputChannel(Token t, Buffer buffer,
+				ConnectionInfo conInfo) {
+			return new TCPInputChannel(buffer, conProvider, conInfo,
+					t.toString(), 0);
+		}
+
+		@Override
+		public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
+				ConnectionInfo conInfo) {
+			return new TCPOutputChannel(buffer, conProvider, conInfo,
+					t.toString(), 0);
+		}
+	}
+
+	/**
+	 * Makes blocking {@link TCPInputChannel} and asynchronous
+	 * {@link AsyncTCPOutputChannel}.
+	 * 
+	 */
+	public class AsyncBoundaryChannelFactory extends TCPBoundaryChannelFactory {
+
+		public AsyncBoundaryChannelFactory(TCPConnectionProvider conProvider) {
+			super(conProvider);
+		}
+
+		@Override
+		public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
+				ConnectionInfo conInfo) {
+			return new AsyncTCPOutputChannel(conProvider, conInfo,
+					t.toString(), 0);
+		}
+	}
+}
\ No newline at end of file

From 6919e01e7166960a80d42d988c98c38ae1708a44 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 May 2014 08:25:28 +0800
Subject: [PATCH 253/881] Gets boundary channels from boundarychannelfactory.

---
 .../impl/distributed/node/BlobsManagerImpl.java   | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index fc996bf0..abf14a34 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -28,6 +28,8 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory.*;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
@@ -52,7 +54,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private Map<Token, BlobExecuter> blobExecuters;
 	private final StreamNode streamNode;
-	private final TCPConnectionProvider conProvider;
+	private final BoundaryChannelFactory chnlFactory;
 	private Map<Token, ConnectionInfo> conInfoMap;
 
 	private Set<Token> globalOutputTokens;
@@ -79,7 +81,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			TCPConnectionProvider conProvider) {
 		this.conInfoMap = conInfoMap;
 		this.streamNode = streamNode;
-		this.conProvider = conProvider;
+		this.chnlFactory = new TCPBoundaryChannelFactory(conProvider);
 
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
@@ -233,8 +235,8 @@ private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 		ImmutableMap.Builder<Token, BoundaryInputChannel> inputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : inputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
-			inputChannelMap.put(t, new TCPInputChannel(bufferMap.get(t),
-					conProvider, conInfo, t.toString(), 0));
+			inputChannelMap.put(t,
+					chnlFactory.makeInputChannel(t, bufferMap.get(t), conInfo));
 		}
 		return inputChannelMap.build();
 	}
@@ -244,8 +246,9 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 		ImmutableMap.Builder<Token, BoundaryOutputChannel> outputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : outputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
-			outputChannelMap.put(t, new AsyncTCPOutputChannel(conProvider,
-					conInfo, t.toString(), 0));
+			outputChannelMap
+					.put(t, chnlFactory.makeOutputChannel(t, bufferMap.get(t),
+							conInfo));
 		}
 		return outputChannelMap.build();
 	}

From 87ff0ad542f5d275249ccbcb591514a40ca14cab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 May 2014 05:47:44 +0800
Subject: [PATCH 254/881] BoundaryChannelManager has been added.

Different channels (Bloking TCP, Async TCP, ect) need to be handled differently
during start and stop. This manager and its sub classes will handle all such
variations.
---
 .../common/BoundaryChannelManager.java        |  57 ++++++++
 .../distributed/node/BlobsManagerImpl.java    | 128 ++++++----------
 .../distributed/node/ChannelManagers.java     | 137 ++++++++++++++++++
 3 files changed, 236 insertions(+), 86 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
new file mode 100644
index 00000000..67b212ca
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
@@ -0,0 +1,57 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import com.google.common.collect.ImmutableMap;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.*;
+
+/**
+ * Manages set of {@link BoundaryChannel}s.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 28, 2014
+ */
+public interface BoundaryChannelManager {
+
+	void start();
+
+	void waitToStart();
+
+	void waitToStop();
+
+	public interface BoundaryInputChannelManager extends BoundaryChannelManager {
+
+		/**
+		 * In streamJit, a channel can be identified by a {@link Token}.
+		 * 
+		 * @return map of channel {@link Token}, {@link BoundaryInputChannel}
+		 *         handled by this manager.
+		 */
+		ImmutableMap<Token, BoundaryInputChannel> inputChannelsMap();
+
+		/**
+		 * @param stopType
+		 *            See {@link BoundaryInputChannel#stop(int)}
+		 */
+		void stop(int stopType);
+	}
+
+	public interface BoundaryOutputChannelManager
+			extends
+				BoundaryChannelManager {
+
+		/**
+		 * In streamJit, a channel can be identified by a {@link Token}.
+		 * 
+		 * @return map of channel {@link Token}, {@link BoundaryOutputChannel}
+		 *         handled by this manager.
+		 */
+		ImmutableMap<Token, BoundaryOutputChannel> outputChannelsMap();
+
+		/**
+		 * @param stopType
+		 *            See {@link BoundaryOutputChannel#stop(boolean)}
+		 */
+		void stop(boolean stopType);
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index abf14a34..b4605fad 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -17,19 +17,18 @@
 import com.google.common.collect.Sets;
 
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.blob.AsyncTCPBuffer;
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory.*;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.*;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
@@ -57,8 +56,6 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final BoundaryChannelFactory chnlFactory;
 	private Map<Token, ConnectionInfo> conInfoMap;
 
-	private Set<Token> globalOutputTokens;
-
 	private MonitorBuffers monBufs;
 
 	private volatile DrainDeadLockHandler drainDeadLockHandler;
@@ -90,6 +87,10 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 
 		bufferMap = createBufferMap(blobSet);
 
+		for (Blob b : blobSet) {
+			b.installBuffers(bufferMap);
+		}
+
 		Set<Token> locaTokens = getLocalTokens(blobSet);
 		blobExecuters = new HashMap<>();
 		for (Blob b : blobSet) {
@@ -162,8 +163,8 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 				minOutputBufCapaciy.keySet());
 		Set<Token> globalInputTokens = Sets.difference(
 				minInputBufCapaciy.keySet(), localTokens);
-		globalOutputTokens = Sets.difference(minOutputBufCapaciy.keySet(),
-				localTokens);
+		Set<Token> globalOutputTokens = Sets.difference(
+				minOutputBufCapaciy.keySet(), localTokens);
 
 		for (Token t : localTokens) {
 			int bufSize = Math.max(minInputBufCapaciy.get(t),
@@ -176,6 +177,10 @@ private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
 			addBuffer(t, bufSize, bufferMapBuilder);
 		}
 
+		for (Token t : globalOutputTokens) {
+			int bufSize = minOutputBufCapaciy.get(t);
+			addBuffer(t, bufSize, bufferMapBuilder);
+		}
 		return bufferMapBuilder.build();
 	}
 
@@ -262,11 +267,8 @@ private class BlobExecuter {
 		private Blob blob;
 		private Set<BlobThread2> blobThreads;
 
-		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
-		private final ImmutableMap<Token, BoundaryOutputChannel> outputChannels;
-
-		Set<Thread> inputChannelThreads;
-		Set<Thread> outputChannelThreads;
+		private final BoundaryInputChannelManager inChnlManager;
+		private final BoundaryOutputChannelManager outChnlManager;
 
 		private boolean reqDrainData;
 
@@ -278,10 +280,10 @@ private BlobExecuter(Token t, Blob blob,
 			this.blobThreads = new HashSet<>();
 			assert blob.getInputs().containsAll(inputChannels.keySet());
 			assert blob.getOutputs().containsAll(outputChannels.keySet());
-			this.inputChannels = inputChannels;
-			this.outputChannels = outputChannels;
-			inputChannelThreads = new HashSet<>(inputChannels.values().size());
-			outputChannelThreads = new HashSet<>(outputChannels.values().size());
+			this.inChnlManager = new ChannelManagers.BlockingInputChannelManager(
+					inputChannels);
+			this.outChnlManager = new ChannelManagers.BlockingOutputChannelManager(
+					outputChannels);
 
 			for (int i = 0; i < blob.getCoreCount(); i++) {
 				StringBuilder sb = new StringBuilder("Workers-");
@@ -301,50 +303,13 @@ private BlobExecuter(Token t, Blob blob,
 		}
 
 		private void startChannels() {
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
-				Thread t = new Thread(bc.getRunnable(), bc.name());
-				t.start();
-				outputChannelThreads.add(t);
-			}
-
-			for (BoundaryInputChannel bc : inputChannels.values()) {
-				Thread t = new Thread(bc.getRunnable(), bc.name());
-				t.start();
-				inputChannelThreads.add(t);
-			}
+			outChnlManager.start();
+			inChnlManager.start();
 		}
 
 		private void start() {
-
-			for (Thread t : outputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-
-			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-					.builder();
-			for (Token t : blob.getInputs()) {
-				bufferMapBuilder.put(t, bufferMap.get(t));
-			}
-
-			for (Token t : blob.getOutputs()) {
-				if (globalOutputTokens.contains(t)) {
-					AsyncTCPOutputChannel asyChannel = (AsyncTCPOutputChannel) outputChannels
-							.get(t);
-					Buffer b = new AsyncTCPBuffer(
-							(AsynchronousTCPConnection) asyChannel
-									.getConnection());
-					bufferMapBuilder.put(t, b);
-				} else {
-					Buffer b = bufferMap.get(t);
-					bufferMapBuilder.put(t, b);
-				}
-			}
-
-			blob.installBuffers(bufferMapBuilder.build());
+			outChnlManager.waitToStart();
+			inChnlManager.waitToStart();
 
 			for (Thread t : blobThreads)
 				t.start();
@@ -353,14 +318,8 @@ private void start() {
 		}
 
 		private void stop() {
-
-			for (BoundaryInputChannel bc : inputChannels.values()) {
-				bc.stop(1);
-			}
-
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
-				bc.stop(true);
-			}
+			inChnlManager.stop(1);
+			outChnlManager.stop(true);
 
 			for (Thread t : blobThreads) {
 				try {
@@ -394,18 +353,20 @@ private void doDrain(boolean reqDrainData) {
 			this.reqDrainData = reqDrainData;
 			drainState = 1;
 
-			for (BoundaryInputChannel bc : inputChannels.values()) {
-				// TODO: [2014-02-05] rearranged this order to call stop(3)
-				// whenever GlobalConstants.useDrainData is false irrespective
-				// of reqDrainData.
-				if (GlobalConstants.useDrainData)
-					if (!this.reqDrainData)
-						bc.stop(1);
-					else
-						bc.stop(2);
+			int stopType;
+			// TODO: [2014-02-05] rearranged this order to call stop(3)
+			// whenever GlobalConstants.useDrainData is false irrespective
+			// of reqDrainData.
+			if (GlobalConstants.useDrainData)
+				if (!this.reqDrainData)
+					stopType = 1;
 				else
-					bc.stop(3);
-			}
+					stopType = 2;
+			else
+				stopType = 3;
+
+			inChnlManager.stop(stopType);
+			// inChnlManager.waitToStop();
 
 			// TODO: [2014-03-14] I commented following lines to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
@@ -453,17 +414,8 @@ private void drained() {
 				bt.requestStop();
 			}
 
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
-				bc.stop(!this.reqDrainData);
-			}
-
-			for (Thread t : outputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
+			outChnlManager.stop(!this.reqDrainData);
+			outChnlManager.waitToStop();
 
 			if (drainState > 3)
 				return;
@@ -532,6 +484,8 @@ private DrainedData getDrainData() {
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
 
+			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
+					.inputChannelsMap();
 			for (Token t : blob.getInputs()) {
 				if (inputChannels.containsKey(t)) {
 					BoundaryChannel chanl = inputChannels.get(t);
@@ -554,6 +508,8 @@ private DrainedData getDrainData() {
 				}
 			}
 
+			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
+					.outputChannelsMap();
 			for (Token t : blob.getOutputs()) {
 				if (outputChannels.containsKey(t)) {
 					BoundaryChannel chanl = outputChannels.get(t);
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java b/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java
new file mode 100644
index 00000000..141f97ce
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java
@@ -0,0 +1,137 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryInputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryOutputChannelManager;
+
+public class ChannelManagers {
+
+	public static class BlockingInputChannelManager
+			implements
+				BoundaryInputChannelManager {
+
+		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
+
+		private final Set<Thread> inputChannelThreads;
+
+		public BlockingInputChannelManager(
+				final ImmutableMap<Token, BoundaryInputChannel> inputChannels) {
+			this.inputChannels = inputChannels;
+			inputChannelThreads = new HashSet<>(inputChannels.values().size());
+		}
+
+		@Override
+		public void start() {
+			for (BoundaryInputChannel bc : inputChannels.values()) {
+				Thread t = new Thread(bc.getRunnable(), bc.name());
+				t.start();
+				inputChannelThreads.add(t);
+			}
+		}
+
+		@Override
+		public void waitToStart() {
+		}
+
+		@Override
+		public void waitToStop() {
+			for (Thread t : inputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		@Override
+		public void stop(int stopType) {
+			for (BoundaryInputChannel bc : inputChannels.values()) {
+				bc.stop(stopType);
+			}
+		}
+
+		@Override
+		public ImmutableMap<Token, BoundaryInputChannel> inputChannelsMap() {
+			return inputChannels;
+		}
+	}
+
+	public static class BlockingOutputChannelManager
+			implements
+				BoundaryOutputChannelManager {
+
+		protected final ImmutableMap<Token, BoundaryOutputChannel> outputChannels;
+		protected final Set<Thread> outputChannelThreads;
+
+		public BlockingOutputChannelManager(
+				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
+			this.outputChannels = outputChannels;
+			outputChannelThreads = new HashSet<>(outputChannels.values().size());
+		}
+
+		@Override
+		public void start() {
+			for (BoundaryOutputChannel bc : outputChannels.values()) {
+				Thread t = new Thread(bc.getRunnable(), bc.name());
+				t.start();
+				outputChannelThreads.add(t);
+			}
+		}
+
+		@Override
+		public void waitToStart() {
+		}
+
+		@Override
+		public void stop(boolean stopType) {
+			for (BoundaryOutputChannel bc : outputChannels.values()) {
+				bc.stop(stopType);
+			}
+		}
+
+		@Override
+		public void waitToStop() {
+			for (Thread t : outputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		@Override
+		public ImmutableMap<Token, BoundaryOutputChannel> outputChannelsMap() {
+			return outputChannels;
+		}
+	}
+
+	public static class AsynchronousOutputChannelManager
+			extends
+				BlockingOutputChannelManager {
+
+		public AsynchronousOutputChannelManager(
+				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
+			super(outputChannels);
+		}
+
+		@Override
+		public void waitToStart() {
+			for (Thread t : outputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+}

From a479b28b360e1e1b6e617fc1fd1e8218e6cef5b7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 30 May 2014 09:50:44 +0800
Subject: [PATCH 255/881] Changes due to cherry-pick and rebase

Closing boundary channels and waiting for corresponding threads to join() have
been moved to boundary channel manager.
---
 .../impl/distributed/node/BlobsManagerImpl.java  | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index b4605fad..943c43b9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -329,20 +329,8 @@ private void stop() {
 				}
 			}
 
-			for (Thread t : inputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-			for (Thread t : outputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
+			inChnlManager.waitToStop();
+			outChnlManager.waitToStop();
 
 			if (monBufs != null)
 				monBufs.stopMonitoring();

From 2012b1992d4dbdfd36e6c08b05b9a5b6fe6eb497 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Jun 2014 10:08:03 +0800
Subject: [PATCH 256/881] Bug Fix : Verifies the number of BlobThreads

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 943c43b9..cda289fc 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -298,6 +298,9 @@ private BlobExecuter(Token t, Blob blob,
 						.toString()));
 			}
 
+			if (blobThreads.size() < 1)
+				throw new IllegalStateException("No blobs to execute");
+
 			drainState = 0;
 			this.blobID = t;
 		}

From aff46ab6d5566d70cfc4551a6e7075fc4156e1b3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Jun 2014 12:29:22 +0800
Subject: [PATCH 257/881] Assigns proper name for BlobThreads.

getName() method returns a base name for a blob and then each thread of
each coreCode will be named as baseName+coreCodeNumber.
---
 .../distributed/node/BlobsManagerImpl.java    | 32 +++++++++++++------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index cda289fc..a25165e0 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -285,17 +285,11 @@ private BlobExecuter(Token t, Blob blob,
 			this.outChnlManager = new ChannelManagers.BlockingOutputChannelManager(
 					outputChannels);
 
+			String baseName = getName(blob);
 			for (int i = 0; i < blob.getCoreCount(); i++) {
-				StringBuilder sb = new StringBuilder("Workers-");
-				int limit = 0;
-				for (Worker<?, ?> w : blob.getWorkers()) {
-					sb.append(Workers.getIdentifier(w));
-					sb.append(",");
-					if (++limit > 5)
-						break;
-				}
-				blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, sb
-						.toString()));
+				String name = String.format("%s - %d", baseName, i);
+				blobThreads
+						.add(new BlobThread2(blob.getCoreCode(i), this, name));
 			}
 
 			if (blobThreads.size() < 1)
@@ -305,6 +299,24 @@ private BlobExecuter(Token t, Blob blob,
 			this.blobID = t;
 		}
 
+		/**
+		 * Returns a name for thread.
+		 * 
+		 * @param blob
+		 * @return
+		 */
+		private String getName(Blob blob) {
+			StringBuilder sb = new StringBuilder("Workers-");
+			int limit = 0;
+			for (Worker<?, ?> w : blob.getWorkers()) {
+				sb.append(Workers.getIdentifier(w));
+				sb.append(",");
+				if (++limit > 5)
+					break;
+			}
+			return sb.toString();
+		}
+
 		private void startChannels() {
 			outChnlManager.start();
 			inChnlManager.start();

From 5edcda7ccc2d2b0a6dd2cb2d21f3090a2c5f9c42 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 02:16:28 +0800
Subject: [PATCH 258/881] AsyncTCPBuffer has been moved into AsyncTCPConnection

---
 .../streamjit/impl/blob/AsyncTCPBuffer.java   | 44 -------------------
 .../common/AsynchronousTCPConnection.java     | 40 +++++++++++++++++
 2 files changed, 40 insertions(+), 44 deletions(-)
 delete mode 100644 src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java

diff --git a/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java b/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
deleted file mode 100644
index 99afc30e..00000000
--- a/src/edu/mit/streamjit/impl/blob/AsyncTCPBuffer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package edu.mit.streamjit.impl.blob;
-
-import java.io.IOException;
-
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
-
-public class AsyncTCPBuffer extends AbstractWriteOnlyBuffer {
-
-	private final AsynchronousTCPConnection con;
-
-	public AsyncTCPBuffer(AsynchronousTCPConnection con) {
-		this.con = con;
-	}
-
-	@Override
-	public boolean write(Object t) {
-		try {
-			con.writeObject(t);
-			return true;
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		return false;
-	}
-
-	public int write(Object[] data, int offset, int length) {
-		try {
-			return con.write(data, offset, length);
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		return 0;
-	}
-
-	@Override
-	public int size() {
-		return 0;
-	}
-
-	@Override
-	public int capacity() {
-		return Integer.MAX_VALUE;
-	}
-}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 35439744..a311d3b8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -13,6 +13,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
+import edu.mit.streamjit.impl.blob.AbstractWriteOnlyBuffer;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -747,4 +748,43 @@ else if (dstID == nodeID) {
 			return con;
 		}
 	}
+
+	public static class AsyncTCPBuffer extends AbstractWriteOnlyBuffer {
+
+		private final AsynchronousTCPConnection con;
+
+		public AsyncTCPBuffer(AsynchronousTCPConnection con) {
+			this.con = con;
+		}
+
+		@Override
+		public boolean write(Object t) {
+			try {
+				con.writeObject(t);
+				return true;
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			return false;
+		}
+
+		public int write(Object[] data, int offset, int length) {
+			try {
+				return con.write(data, offset, length);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			return 0;
+		}
+
+		@Override
+		public int size() {
+			return 0;
+		}
+
+		@Override
+		public int capacity() {
+			return Integer.MAX_VALUE;
+		}
+	}
 }

From 347dd91b425e26d0cda803d75feaccbbff1e7957 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 02:24:15 +0800
Subject: [PATCH 259/881] BoundaryChannels return Buffer it handles

getBuffer() method will returns the buffer that is handled by a
BoundaryChannel.
---
 .../impl/distributed/common/BoundaryChannel.java     |  2 ++
 .../impl/distributed/node/AsyncTCPOutputChannel.java | 12 ++++++++++++
 .../impl/distributed/node/TCPInputChannel.java       |  5 +++++
 .../impl/distributed/node/TCPOutputChannel.java      |  5 +++++
 4 files changed, 24 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
index 1ebc4bf9..cf1660c6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
@@ -31,6 +31,8 @@ public interface BoundaryChannel {
 
 	ConnectionInfo getConnectionInfo();
 
+	Buffer getBuffer();
+
 	/**
 	 * Interface that represents input channels.
 	 */
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 960ed741..6769ef7f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -4,6 +4,9 @@
 
 import com.google.common.collect.ImmutableList;
 
+import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPBuffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
@@ -19,6 +22,8 @@ public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
 	private final ConnectionInfo conInfo;
 
+	private AsyncTCPBuffer buffer = null;
+
 	private volatile boolean isFinal;
 
 	private volatile boolean stopCalled;
@@ -45,6 +50,8 @@ public void run() {
 				if (con == null || !con.isStillConnected()) {
 					try {
 						con = conProvider.getConnection(conInfo);
+						buffer = new AsyncTCPBuffer(
+								(AsynchronousTCPConnection) con);
 					} catch (IOException e) {
 						e.printStackTrace();
 					}
@@ -85,4 +92,9 @@ public Connection getConnection() {
 	public ConnectionInfo getConnectionInfo() {
 		return conInfo;
 	}
+
+	@Override
+	public Buffer getBuffer() {
+		return buffer;
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index f8614481..10968b2d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -446,4 +446,9 @@ public ConnectionInfo getConnectionInfo() {
 		// TODO Auto-generated method stub
 		return null;
 	}
+
+	@Override
+	public Buffer getBuffer() {
+		return buffer;
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index d1b0293d..68b95fbb 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -247,4 +247,9 @@ public Connection getConnection() {
 	public ConnectionInfo getConnectionInfo() {
 		return conInfo;
 	}
+
+	@Override
+	public Buffer getBuffer() {
+		return buffer;
+	}
 }

From c79d8aea3cb57b1ddcd6cae59ec61ddac168a437 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 02:42:36 +0800
Subject: [PATCH 260/881] TCPInput and Output Channels create buffers

---
 .../streamjit/impl/distributed/node/TCPInputChannel.java   | 7 +++++++
 .../streamjit/impl/distributed/node/TCPOutputChannel.java  | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index 10968b2d..cdd0f9bc 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -13,6 +13,7 @@
 
 import edu.mit.streamjit.impl.blob.AbstractBuffer;
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
@@ -60,6 +61,12 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private ImmutableList<Object> unProcessedData;
 
+	public TCPInputChannel(int bufSize, TCPConnectionProvider conProvider,
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
+				bufferTokenName, debugLevel);
+	}
+
 	public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index 68b95fbb..a6a9184a 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -8,6 +8,7 @@
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
@@ -49,6 +50,12 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	protected ImmutableList<Object> unProcessedData;
 
+	public TCPOutputChannel(int bufSize, TCPConnectionProvider conProvider,
+			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
+				bufferTokenName, debugLevel);
+	}
+
 	public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;

From 8e3c69b99a29e203c9c2c2023290079d8f0338d0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 02:46:04 +0800
Subject: [PATCH 261/881] MakeChannels by passing buffer sizes.

---
 .../common/BoundaryChannelFactory.java        | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index dd6d18ed..231d4306 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -24,6 +24,12 @@ BoundaryInputChannel makeInputChannel(Token t, Buffer buffer,
 	BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 			ConnectionInfo conInfo);
 
+	BoundaryInputChannel makeInputChannel(Token t, int bufSize,
+			ConnectionInfo conInfo);
+
+	BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
+			ConnectionInfo conInfo);
+
 	/**
 	 * Makes blocking {@link TCPInputChannel} and {@link TCPOutputChannel}.
 	 * 
@@ -51,6 +57,20 @@ public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 			return new TCPOutputChannel(buffer, conProvider, conInfo,
 					t.toString(), 0);
 		}
+
+		@Override
+		public BoundaryInputChannel makeInputChannel(Token t, int bufSize,
+				ConnectionInfo conInfo) {
+			return new TCPInputChannel(bufSize, conProvider, conInfo,
+					t.toString(), 0);
+		}
+
+		@Override
+		public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
+				ConnectionInfo conInfo) {
+			return new TCPOutputChannel(bufSize, conProvider, conInfo,
+					t.toString(), 0);
+		}
 	}
 
 	/**
@@ -70,5 +90,12 @@ public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 			return new AsyncTCPOutputChannel(conProvider, conInfo,
 					t.toString(), 0);
 		}
+
+		@Override
+		public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
+				ConnectionInfo conInfo) {
+			return new AsyncTCPOutputChannel(conProvider, conInfo,
+					t.toString(), 0);
+		}
 	}
 }
\ No newline at end of file

From 1bc9073dad0b64f1e41af6a8c112eb285fdb1651 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 12:12:44 +0800
Subject: [PATCH 262/881] Interface BufferManager has been added.

Manages Buffer. BlobsManager will use the services from BufferManager.
Implementation of this interface is expected to do two tasks
 1.Calculates buffer sizes.
 2.Create local buffers.
---
 .../impl/distributed/node/BufferManager.java  | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/BufferManager.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
new file mode 100644
index 00000000..fd7975c8
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -0,0 +1,59 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.Blob.Token;
+
+/**
+ * {@link BlobsManager} will use the services from {@link BufferManager}.
+ * Implementation of this interface is expected to do two tasks
+ * <ol>
+ * <li>Calculates buffer sizes.
+ * <li>Create local buffers.
+ * </ol>
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 28, 2014
+ * 
+ */
+public interface BufferManager {
+
+	void initialise();
+
+	/**
+	 * Second initialisation. If the buffer sizes are computed by controller and
+	 * send back to the {@link StreamNode}s, this method can be called with the
+	 * minimum input buffer size requirement.
+	 * 
+	 * @param minInputBufSizes
+	 */
+	void initialise2(Map<Token, Integer> minInputBufSizes);
+
+	ImmutableSet<Token> localTokens();
+
+	ImmutableSet<Token> outputTokens();
+
+	ImmutableSet<Token> inputTokens();
+
+	/**
+	 * @return buffer sizes of each local and boundary channels. Returns
+	 *         <code>null</code> if the buffer sizes are not calculated yet.
+	 *         {@link #isbufferSizesReady()} tells whether the buffer sizes are
+	 *         calculated or not.
+	 */
+	ImmutableMap<Token, Integer> bufferSizes();
+
+	/**
+	 * @return <code>true</code> iff buffer sizes are calculated.
+	 */
+	boolean isbufferSizesReady();
+
+	/**
+	 * @return local buffers
+	 */
+	ImmutableMap<Token, Buffer> localBufferMap();
+}

From d4db0c7cd811f0fbd10f5c63c2420c0b74d8b58e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 13:07:31 +0800
Subject: [PATCH 263/881] AbstractBufferManager has been implemented

---
 .../impl/distributed/node/BufferManager.java  | 76 ++++++++++++++++++-
 1 file changed, 75 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index fd7975c8..ae95ebef 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -1,10 +1,14 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 
+import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 
@@ -53,7 +57,77 @@ public interface BufferManager {
 	boolean isbufferSizesReady();
 
 	/**
-	 * @return local buffers
+	 * @return local buffers if buffer sizes are calculated. Otherwise returns
+	 *         null.
 	 */
 	ImmutableMap<Token, Buffer> localBufferMap();
+
+	public static abstract class AbstractBufferManager implements BufferManager {
+
+		protected final Set<Blob> blobSet;
+
+		protected final ImmutableSet<Token> localTokens;
+
+		protected final ImmutableSet<Token> globalInputTokens;
+
+		protected final ImmutableSet<Token> globalOutputTokens;
+
+		protected boolean isbufferSizesReady;
+
+		protected ImmutableMap<Token, Integer> bufferSizes;
+
+		ImmutableMap<Token, Buffer> localBufferMap;
+
+		public AbstractBufferManager(Set<Blob> blobSet) {
+			this.blobSet = blobSet;
+
+			Set<Token> inputTokens = new HashSet<>();
+			Set<Token> outputTokens = new HashSet<>();
+			for (Blob b : blobSet) {
+				inputTokens.addAll(b.getInputs());
+				outputTokens.addAll(b.getOutputs());
+			}
+
+			localTokens = ImmutableSet.copyOf(Sets.intersection(inputTokens,
+					outputTokens));
+			globalInputTokens = ImmutableSet.copyOf(Sets.difference(
+					inputTokens, localTokens));
+			globalOutputTokens = ImmutableSet.copyOf(Sets.difference(
+					outputTokens, localTokens));
+
+			isbufferSizesReady = false;
+			bufferSizes = null;
+			localBufferMap = null;
+		}
+
+		@Override
+		public ImmutableSet<Token> localTokens() {
+			return localTokens;
+		}
+
+		@Override
+		public ImmutableSet<Token> outputTokens() {
+			return globalOutputTokens;
+		}
+
+		@Override
+		public ImmutableSet<Token> inputTokens() {
+			return globalInputTokens;
+		}
+
+		@Override
+		public ImmutableMap<Token, Integer> bufferSizes() {
+			return bufferSizes;
+		}
+
+		@Override
+		public boolean isbufferSizesReady() {
+			return isbufferSizesReady;
+		}
+
+		@Override
+		public ImmutableMap<Token, Buffer> localBufferMap() {
+			return localBufferMap;
+		}
+	}
 }

From 64a24ef032a1ddd9201c631114c8eb09ea1e3cd7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 13:49:15 +0800
Subject: [PATCH 264/881] LocalBufferManager has been implemented

---
 .../impl/distributed/node/BufferManager.java  | 94 +++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index ae95ebef..c503ac5e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -10,6 +11,7 @@
 
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 
 /**
@@ -129,5 +131,97 @@ public boolean isbufferSizesReady() {
 		public ImmutableMap<Token, Buffer> localBufferMap() {
 			return localBufferMap;
 		}
+
+		protected final void createLocaBuffers() {
+			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+					.<Token, Buffer> builder();
+			for (Token t : localTokens) {
+				int bufSize = bufferSizes.get(t);
+				bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
+			}
+			localBufferMap = bufferMapBuilder.build();
+		}
+	}
+
+	/**
+	 * Calculates buffer sizes locally at {@link StreamNode} side. No central
+	 * calculation involved.
+	 */
+	public class LocalBufferManager extends AbstractBufferManager {
+		public LocalBufferManager(Set<Blob> blobSet) {
+			super(blobSet);
+		}
+
+		@Override
+		public void initialise() {
+			bufferSizes = calculateBufferSizes(blobSet);
+			createLocaBuffers();
+			isbufferSizesReady = true;
+		}
+
+		@Override
+		public void initialise2(Map<Token, Integer> minInputBufSizes) {
+			throw new java.lang.Error(
+					"initialise2() is not supposed to be called");
+		}
+
+		// TODO: Buffer sizes, including head and tail buffers, must be
+		// optimised. consider adding some tuning factor
+		private ImmutableMap<Token, Integer> calculateBufferSizes(
+				Set<Blob> blobSet) {
+			ImmutableMap.Builder<Token, Integer> bufferSizeMapBuilder = ImmutableMap
+					.<Token, Integer> builder();
+
+			Map<Token, Integer> minInputBufCapaciy = new HashMap<>();
+			Map<Token, Integer> minOutputBufCapaciy = new HashMap<>();
+
+			for (Blob b : blobSet) {
+				Set<Blob.Token> inputs = b.getInputs();
+				for (Token t : inputs) {
+					minInputBufCapaciy.put(t, b.getMinimumBufferCapacity(t));
+				}
+
+				Set<Blob.Token> outputs = b.getOutputs();
+				for (Token t : outputs) {
+					minOutputBufCapaciy.put(t, b.getMinimumBufferCapacity(t));
+				}
+			}
+
+			Set<Token> localTokens = Sets.intersection(
+					minInputBufCapaciy.keySet(), minOutputBufCapaciy.keySet());
+			Set<Token> globalInputTokens = Sets.difference(
+					minInputBufCapaciy.keySet(), localTokens);
+			Set<Token> globalOutputTokens = Sets.difference(
+					minOutputBufCapaciy.keySet(), localTokens);
+
+			for (Token t : localTokens) {
+				int bufSize = Math.max(minInputBufCapaciy.get(t),
+						minOutputBufCapaciy.get(t));
+				addBuffer(t, bufSize, bufferSizeMapBuilder);
+			}
+
+			for (Token t : globalInputTokens) {
+				int bufSize = minInputBufCapaciy.get(t);
+				addBuffer(t, bufSize, bufferSizeMapBuilder);
+			}
+
+			for (Token t : globalOutputTokens) {
+				int bufSize = minOutputBufCapaciy.get(t);
+				addBuffer(t, bufSize, bufferSizeMapBuilder);
+			}
+			return bufferSizeMapBuilder.build();
+		}
+
+		/**
+		 * Just introduced to avoid code duplication.
+		 */
+		private void addBuffer(Token t, int minSize,
+				ImmutableMap.Builder<Token, Integer> bufferSizeMapBuilder) {
+			// TODO: Just to increase the performance. Change it later
+			int bufSize = Math.max(1000, minSize);
+			// System.out.println("Buffer size of " + t.toString() + " is " +
+			// bufSize);
+			bufferSizeMapBuilder.put(t, bufSize);
+		}
 	}
 }

From 2a226b295d39eb74d89661777fbca38477afc9fa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 13:50:25 +0800
Subject: [PATCH 265/881] Renaming : addBuffer() -> addBufferSize()

---
 .../streamjit/impl/distributed/node/BufferManager.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index c503ac5e..5ad6d309 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -147,7 +147,7 @@ protected final void createLocaBuffers() {
 	 * Calculates buffer sizes locally at {@link StreamNode} side. No central
 	 * calculation involved.
 	 */
-	public class LocalBufferManager extends AbstractBufferManager {
+	public static class LocalBufferManager extends AbstractBufferManager {
 		public LocalBufferManager(Set<Blob> blobSet) {
 			super(blobSet);
 		}
@@ -197,17 +197,17 @@ private ImmutableMap<Token, Integer> calculateBufferSizes(
 			for (Token t : localTokens) {
 				int bufSize = Math.max(minInputBufCapaciy.get(t),
 						minOutputBufCapaciy.get(t));
-				addBuffer(t, bufSize, bufferSizeMapBuilder);
+				addBufferSize(t, bufSize, bufferSizeMapBuilder);
 			}
 
 			for (Token t : globalInputTokens) {
 				int bufSize = minInputBufCapaciy.get(t);
-				addBuffer(t, bufSize, bufferSizeMapBuilder);
+				addBufferSize(t, bufSize, bufferSizeMapBuilder);
 			}
 
 			for (Token t : globalOutputTokens) {
 				int bufSize = minOutputBufCapaciy.get(t);
-				addBuffer(t, bufSize, bufferSizeMapBuilder);
+				addBufferSize(t, bufSize, bufferSizeMapBuilder);
 			}
 			return bufferSizeMapBuilder.build();
 		}
@@ -215,7 +215,7 @@ private ImmutableMap<Token, Integer> calculateBufferSizes(
 		/**
 		 * Just introduced to avoid code duplication.
 		 */
-		private void addBuffer(Token t, int minSize,
+		private void addBufferSize(Token t, int minSize,
 				ImmutableMap.Builder<Token, Integer> bufferSizeMapBuilder) {
 			// TODO: Just to increase the performance. Change it later
 			int bufSize = Math.max(1000, minSize);

From 285ef1be5b60c10d47382d7a87571cbb014e5a29 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Jun 2014 16:01:40 +0800
Subject: [PATCH 266/881] Restructured : Uses BufferManager

Buffer size calculation and buffer creation have been moved away from
BlobsManager.
---
 .../distributed/node/BlobsManagerImpl.java    | 157 +++++++++---------
 1 file changed, 75 insertions(+), 82 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index a25165e0..24db18c8 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -19,7 +19,6 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
@@ -41,6 +40,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
 
 /**
  * {@link BlobsManagerImpl} responsible to run all {@link Blob}s those are
@@ -64,7 +64,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final CommandProcessor cmdProcessor;
 
-	private final ImmutableMap<Token, Buffer> bufferMap;
+	private final BufferManager bufferManager;
 
 	/**
 	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
@@ -84,21 +84,24 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
 		this.drainDeadLockHandler = null;
 		this.useDrainDeadLockHandler = false;
+		this.bufferManager = new LocalBufferManager(blobSet);
 
-		bufferMap = createBufferMap(blobSet);
-
-		for (Blob b : blobSet) {
-			b.installBuffers(bufferMap);
-		}
+		bufferManager.initialise();
+		if (bufferManager.isbufferSizesReady())
+			createBEs(blobSet);
+	}
 
-		Set<Token> locaTokens = getLocalTokens(blobSet);
+	private void createBEs(ImmutableSet<Blob> blobSet) {
 		blobExecuters = new HashMap<>();
+		Set<Token> locaTokens = bufferManager.localTokens();
+		ImmutableMap<Token, Integer> bufferSizesMap = bufferManager
+				.bufferSizes();
 		for (Blob b : blobSet) {
 			Token t = Utils.getBlobID(b);
 			ImmutableMap<Token, BoundaryInputChannel> inputChannels = createInputChannels(
-					Sets.difference(b.getInputs(), locaTokens), bufferMap);
+					Sets.difference(b.getInputs(), locaTokens), bufferSizesMap);
 			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = createOutputChannels(
-					Sets.difference(b.getOutputs(), locaTokens), bufferMap);
+					Sets.difference(b.getOutputs(), locaTokens), bufferSizesMap);
 			blobExecuters.put(t, new BlobExecuter(t, b, inputChannels,
 					outputChannels));
 		}
@@ -138,67 +141,6 @@ public void stop() {
 		if (drainDeadLockHandler != null)
 			drainDeadLockHandler.stopit();
 	}
-	// TODO: Buffer sizes, including head and tail buffers, must be optimized.
-	// consider adding some tuning factor
-	private ImmutableMap<Token, Buffer> createBufferMap(Set<Blob> blobSet) {
-		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-				.<Token, Buffer> builder();
-
-		Map<Token, Integer> minInputBufCapaciy = new HashMap<>();
-		Map<Token, Integer> minOutputBufCapaciy = new HashMap<>();
-
-		for (Blob b : blobSet) {
-			Set<Blob.Token> inputs = b.getInputs();
-			for (Token t : inputs) {
-				minInputBufCapaciy.put(t, b.getMinimumBufferCapacity(t));
-			}
-
-			Set<Blob.Token> outputs = b.getOutputs();
-			for (Token t : outputs) {
-				minOutputBufCapaciy.put(t, b.getMinimumBufferCapacity(t));
-			}
-		}
-
-		Set<Token> localTokens = Sets.intersection(minInputBufCapaciy.keySet(),
-				minOutputBufCapaciy.keySet());
-		Set<Token> globalInputTokens = Sets.difference(
-				minInputBufCapaciy.keySet(), localTokens);
-		Set<Token> globalOutputTokens = Sets.difference(
-				minOutputBufCapaciy.keySet(), localTokens);
-
-		for (Token t : localTokens) {
-			int bufSize = Math.max(minInputBufCapaciy.get(t),
-					minOutputBufCapaciy.get(t));
-			addBuffer(t, bufSize, bufferMapBuilder);
-		}
-
-		for (Token t : globalInputTokens) {
-			int bufSize = minInputBufCapaciy.get(t);
-			addBuffer(t, bufSize, bufferMapBuilder);
-		}
-
-		for (Token t : globalOutputTokens) {
-			int bufSize = minOutputBufCapaciy.get(t);
-			addBuffer(t, bufSize, bufferMapBuilder);
-		}
-		return bufferMapBuilder.build();
-	}
-
-	/**
-	 * Just introduced to avoid code duplication.
-	 * 
-	 * @param t
-	 * @param minSize
-	 * @param bufferMapBuilder
-	 */
-	private void addBuffer(Token t, int minSize,
-			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder) {
-		// TODO: Just to increase the performance. Change it later
-		int bufSize = Math.max(1000, minSize);
-		// System.out.println("Buffer size of " + t.toString() + " is " +
-		// bufSize);
-		bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
-	}
 
 	private long gcd(long a, long b) {
 		while (true) {
@@ -236,7 +178,7 @@ private Set<Token> getLocalTokens(Set<Blob> blobSet) {
 	}
 
 	private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
-			Set<Token> inputTokens, ImmutableMap<Token, Buffer> bufferMap) {
+			Set<Token> inputTokens, ImmutableMap<Token, Integer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryInputChannel> inputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : inputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
@@ -247,7 +189,7 @@ private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 	}
 
 	private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
-			Set<Token> outputTokens, ImmutableMap<Token, Buffer> bufferMap) {
+			Set<Token> outputTokens, ImmutableMap<Token, Integer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryOutputChannel> outputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : outputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
@@ -272,6 +214,8 @@ private class BlobExecuter {
 
 		private boolean reqDrainData;
 
+		private ImmutableMap<Token, Buffer> bufferMap;
+
 		private BlobExecuter(Token t, Blob blob,
 				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
 				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
@@ -326,6 +270,9 @@ private void start() {
 			outChnlManager.waitToStart();
 			inChnlManager.waitToStart();
 
+			bufferMap = buildBufferMap();
+			blob.installBuffers(bufferMap);
+
 			for (Thread t : blobThreads)
 				t.start();
 
@@ -546,6 +493,44 @@ private DrainedData getEmptyDrainData() {
 		public Token getBlobID() {
 			return blobID;
 		}
+
+		private ImmutableMap<Token, Buffer> buildBufferMap() {
+			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+					.builder();
+			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
+					.localBufferMap();
+			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
+					.inputChannelsMap();
+			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
+					.outputChannelsMap();
+
+			for (Token t : blob.getInputs()) {
+				if (localBufferMap.containsKey(t)) {
+					assert !inputChannels.containsKey(t) : "Same channels is exists in both localBuffer and inputChannel";
+					bufferMapBuilder.put(t, localBufferMap.get(t));
+				} else if (inputChannels.containsKey(t)) {
+					BoundaryInputChannel chnl = inputChannels.get(t);
+					bufferMapBuilder.put(t, chnl.getBuffer());
+				} else {
+					throw new AssertionError(String.format(
+							"No Buffer for input channel %s ", t));
+				}
+			}
+
+			for (Token t : blob.getOutputs()) {
+				if (localBufferMap.containsKey(t)) {
+					assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
+					bufferMapBuilder.put(t, localBufferMap.get(t));
+				} else if (outputChannels.containsKey(t)) {
+					BoundaryOutputChannel chnl = outputChannels.get(t);
+					bufferMapBuilder.put(t, chnl.getBuffer());
+				} else {
+					throw new AssertionError(String.format(
+							"No Buffer for output channel %s ", t));
+				}
+			}
+			return bufferMapBuilder.build();
+		}
 	}
 
 	private static class DrainCallback implements Runnable {
@@ -782,17 +767,25 @@ public void run() {
 						Thread.sleep(sleepTime);
 					} catch (InterruptedException e) {
 					}
-					if (bufferMap == null) {
+					if (blobExecuters == null) {
 						writter.write("Buffer map is null...\n");
 						continue;
 					}
-					if (stopFlag.get())
-						break;
 					writter.write("----------------------------------\n");
-					for (Map.Entry<Token, Buffer> en : bufferMap.entrySet()) {
-						writter.write(en.getKey() + " - "
-								+ en.getValue().size());
-						writter.write('\n');
+					for (BlobExecuter be : blobExecuters.values()) {
+						if (be.bufferMap == null) {
+							writter.write("Buffer map is null...\n");
+							continue;
+						}
+						if (stopFlag.get())
+							break;
+
+						for (Map.Entry<Token, Buffer> en : be.bufferMap
+								.entrySet()) {
+							writter.write(en.getKey() + " - "
+									+ en.getValue().size());
+							writter.write('\n');
+						}
 					}
 					writter.write("----------------------------------\n");
 					writter.flush();
@@ -846,7 +839,7 @@ public void run() {
 						// System.out.println(be.blobID + " is not drained");
 						areAllDrained = false;
 						for (Token t : be.blob.getOutputs()) {
-							Buffer b = bufferMap.get(t);
+							Buffer b = be.bufferMap.get(t);
 							int size = b.size();
 							if (size == 0)
 								continue;
@@ -857,7 +850,7 @@ public void run() {
 							b.readAll(obArray);
 						}
 						for (Token t : be.blob.getInputs()) {
-							Buffer b = bufferMap.get(t);
+							Buffer b = be.bufferMap.get(t);
 							int size = b.size();
 							if (size == 0)
 								continue;

From 0fd2ca976cf2a07627170df54157889e849fbd59 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Jun 2014 10:08:29 +0800
Subject: [PATCH 267/881] Obselete methods have been removed

gcd(), lcm() and getLocalTokens() have been removed.
---
 .../distributed/node/BlobsManagerImpl.java    | 35 -------------------
 1 file changed, 35 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 24db18c8..0493a1fb 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -142,41 +142,6 @@ public void stop() {
 			drainDeadLockHandler.stopit();
 	}
 
-	private long gcd(long a, long b) {
-		while (true) {
-			if (a == 0)
-				return b;
-			b %= a;
-			if (b == 0)
-				return a;
-			a %= b;
-		}
-	}
-
-	private long lcm(long a, long b) {
-		long val = gcd(a, b);
-		long quotient = a / val;
-		return val != 0 ? b * quotient : 0;
-	}
-
-	private Set<Token> getLocalTokens(Set<Blob> blobSet) {
-		Set<Token> inputTokens = new HashSet<>();
-		Set<Token> outputTokens = new HashSet<>();
-
-		for (Blob b : blobSet) {
-			Set<Token> inputs = b.getInputs();
-			for (Token t : inputs) {
-				inputTokens.add(t);
-			}
-
-			Set<Token> outputs = b.getOutputs();
-			for (Token t : outputs) {
-				outputTokens.add(t);
-			}
-		}
-		return Sets.intersection(inputTokens, outputTokens);
-	}
-
 	private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 			Set<Token> inputTokens, ImmutableMap<Token, Integer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryInputChannel> inputChannelMap = new ImmutableMap.Builder<>();

From b5ddd1b2c5c6d2ee35e41eec402da814b2fd6618 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 18 Jun 2014 01:44:32 +0800
Subject: [PATCH 268/881] Comment is added to state the purpose a variable

Comments tell the purpose of the instance variable 'crashed'.
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 0493a1fb..86781a28 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -40,6 +40,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
 
 /**
@@ -167,6 +168,12 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 
 	private class BlobExecuter {
 
+		/**
+		 * This flag will be set to true if an exception thrown by the core code
+		 * of the {@link Blob}. Any exception occurred in a blob's corecode will
+		 * be informed to {@link Controller} to halt the application. See the
+		 * {@link BlobThread2}.
+		 */
 		private AtomicBoolean crashed;
 		private volatile int drainState;
 		private final Token blobID;

From 61620d6385374ee62f87d118b278c160f4389516 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 18 Jun 2014 09:33:11 +0800
Subject: [PATCH 269/881] Stops execution if no cfg file

In the 'no tuning' case, halts the execution if the configuration file is not
available in the current running directory.
---
 .../impl/distributed/DistributedStreamCompiler.java          | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 0a09ba4b..4418fbe6 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -143,7 +143,10 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		if (GlobalConstants.tune == 0) {
 			Configuration cfg1 = readConfiguration(stream.getClass()
 					.getSimpleName());
-			if (!this.cfg.getParametersMap().keySet()
+			if (cfg1 == null) {
+				controller.closeAll();
+				throw new IllegalConfigurationException();
+			} else if (!this.cfg.getParametersMap().keySet()
 					.equals(cfg1.getParametersMap().keySet())) {
 				System.err
 						.println("Reading the configuration from configuration file");

From 263aaf893d39aca0fd0b83389c12f7a5883c7dd2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 19 Jun 2014 16:09:58 +0800
Subject: [PATCH 270/881] Stops thread that logs performance.

We need to stop PerformanceLogger thread when the input is finished and
draining is completed while the PerformanceLogger thread is running.
---
 .../streamjit/impl/distributed/StreamJitAppManager.java  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index ecf0f923..dcf6b339 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -279,10 +279,9 @@ else if (GlobalConstants.useDrainData)
 			}
 		}
 
-		if (isFinal) {
-			this.status = AppStatus.STOPPED;
-			controller.closeAll();
-		}
+		if (isFinal)
+			stop();
+
 		isRunning = false;
 
 		Stopwatch sw = stopwatchRef.get();
@@ -333,7 +332,7 @@ private void reset() {
 
 	public void stop() {
 		this.status = AppStatus.STOPPED;
-		tailChannel.reset();
+		tailChannel.releaseAll();
 		controller.closeAll();
 		dp.drainer.stop();
 	}

From d0c9b558243281f7caa862d216d1b5cdb9806e38 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Jun 2014 16:46:30 +0800
Subject: [PATCH 271/881] AsynchronousTCPHeadChannel has been added

Global input data can be sent to workers in Asynchronous mode too.
---
 .../impl/distributed/HeadChannel.java         | 76 +++++++++++++++++--
 .../impl/distributed/StreamJitAppManager.java |  2 +-
 2 files changed, 71 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 9844428e..6176f413 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -7,6 +7,7 @@
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
 
 /**
@@ -16,15 +17,59 @@
  * @author Sumanan sumanan@mit.edu
  * @since Oct 21, 2013
  */
-public class HeadChannel extends TCPOutputChannel {
+public class HeadChannel {
 
-	public HeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
-			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
+	public static class TCPHeadChannel extends TCPOutputChannel {
+
+		public TCPHeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
+				ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
+			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
+		}
+
+		protected void fillUnprocessedData() {
+			this.unProcessedData = ImmutableList.of();
+		}
 	}
 
-	protected void fillUnprocessedData() {
-		this.unProcessedData = ImmutableList.of();
+	public static class AsynchTCPHeadChannel extends AsyncTCPOutputChannel {
+
+		final Buffer readBuffer;
+		private volatile boolean stopCalled;
+		public AsynchTCPHeadChannel(Buffer buffer,
+				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
+				String bufferTokenName, int debugLevel) {
+			super(conProvider, conInfo, bufferTokenName, debugLevel);
+			readBuffer = buffer;
+			stopCalled = false;
+		}
+
+		@Override
+		public Runnable getRunnable() {
+			final Runnable supperRunnable = super.getRunnable();
+			return new Runnable() {
+				@Override
+				public void run() {
+					supperRunnable.run();
+					final Buffer writeBuffer = getBuffer();
+					final int dataLength = 10000;
+					final Object[] data = new Object[dataLength];
+					int read = 1;
+					while (read != 0 && !stopCalled) {
+						read = readBuffer.read(data, 0, data.length);
+						writeBuffer.write(data, 0, read);
+					}
+				}
+			};
+		}
+		protected void fillUnprocessedData() {
+			throw new Error("Method not implemented");
+		}
+
+		@Override
+		public void stop(boolean isFinal) {
+			super.stop(isFinal);
+			this.stopCalled = true;
+		}
 	}
 
 	/**
@@ -56,6 +101,25 @@ public Object read() {
 			return o;
 		}
 
+		@Override
+		public int read(Object[] data, int offset, int length) {
+			int read = buffer.read(data, offset, length);
+			if (read == 0) {
+				new DrainerThread().start();
+			}
+			return read;
+		}
+
+		@Override
+		public boolean readAll(Object[] data) {
+			return buffer.readAll(data);
+		}
+
+		@Override
+		public boolean readAll(Object[] data, int offset) {
+			return buffer.readAll(data, offset);
+		}
+
 		@Override
 		public int size() {
 			return buffer.size();
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index dcf6b339..f10cb7a1 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -177,7 +177,7 @@ private void setupHeadTail(Map<Token, ConnectionInfo> conInfoMap,
 			throw new IllegalArgumentException(
 					"No head buffer in the passed bufferMap.");
 
-		headChannel = new HeadChannel(bufferMap.get(headToken),
+		headChannel = new HeadChannel.TCPHeadChannel(bufferMap.get(headToken),
 				controller.getConProvider(), headconInfo, "headChannel - "
 						+ headToken.toString(), 0);
 

From 9c86ea5c6a3412b24b75994fd64a20227fef2c76 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 23 Jun 2014 13:38:33 +0800
Subject: [PATCH 272/881] bulk read added to LoopedBufferBuffer

Bulk reading has been made more efficient. AsynchronousTCPChannel uses
bulk reading to read and send the data.
---
 src/edu/mit/streamjit/test/Datasets.java | 38 ++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/src/edu/mit/streamjit/test/Datasets.java b/src/edu/mit/streamjit/test/Datasets.java
index 0de1a826..98c0a2fe 100644
--- a/src/edu/mit/streamjit/test/Datasets.java
+++ b/src/edu/mit/streamjit/test/Datasets.java
@@ -71,6 +71,44 @@ public Object read() {
 			increment(1);
 			return ret;
 		}
+
+		@Override
+		public int read(Object[] data, int offset, int length) {
+			int read = 0;
+			int min = Math.min(size(), length);
+			Object obj;
+			while (read < min && (obj = peek(read)) != null) {
+				data[offset++] = obj;
+				++read;
+			}
+			increment(read);
+			return read;
+		}
+
+		@Override
+		public boolean readAll(Object[] data) {
+			return readAll(data, 0);
+		}
+
+		@Override
+		public boolean readAll(Object[] data, int offset) {
+			int required = data.length - offset;
+			if (required > size())
+				return false;
+			int read;
+			for (read = 0; read < required; ++offset, ++read) {
+				Object e = peek(read);
+				// We checked size() above, so we should never fail here, except
+				// in
+				// case of concurrent modification by another reader.
+				assert e != null;
+				data[offset] = e;
+			}
+			increment(read);
+			assert (data.length == offset) : "data.length != offset - Check the arithmetic";
+			return true;
+		}
+
 		@Override
 		public int size() {
 			if (cycles < 0) return 0;

From 579de959ae066a1432ea34bfd526b56cecc57473 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 23 Jun 2014 16:55:54 +0800
Subject: [PATCH 273/881] Obsolete code removed

Extra configuration parameters portIDMap and tokenMachineMap have been removed.
---
 .../streamjit/impl/distributed/StreamJitAppManager.java    | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index f10cb7a1..174dae12 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -1,6 +1,5 @@
 package edu.mit.streamjit.impl.distributed;
 
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -110,15 +109,9 @@ public boolean reconfigure(int multiplier) {
 		Configuration.Builder builder = Configuration.builder(cfgManager
 				.getDynamicConfiguration());
 
-		Map<Token, Map.Entry<Integer, Integer>> tokenMachineMap = new HashMap<>();
-		Map<Token, Integer> portIdMap = new HashMap<>();
-
 		conInfoMap = controller.buildConInfoMap(app.partitionsMachineMap,
 				app.source, app.sink);
 
-		builder.putExtraData(GlobalConstants.TOKEN_MACHINE_MAP, tokenMachineMap)
-				.putExtraData(GlobalConstants.PORTID_MAP, portIdMap);
-
 		builder.putExtraData(GlobalConstants.CONINFOMAP, conInfoMap);
 
 		Configuration cfg = builder.build();

From 3d0c6e160253e614cccf4d53bb4325b784ab2e94 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Jun 2014 11:51:02 +0800
Subject: [PATCH 274/881] Asynch -> Async

---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 6176f413..043438f5 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -31,11 +31,11 @@ protected void fillUnprocessedData() {
 		}
 	}
 
-	public static class AsynchTCPHeadChannel extends AsyncTCPOutputChannel {
+	public static class AsyncTCPHeadChannel extends AsyncTCPOutputChannel {
 
 		final Buffer readBuffer;
 		private volatile boolean stopCalled;
-		public AsynchTCPHeadChannel(Buffer buffer,
+		public AsyncTCPHeadChannel(Buffer buffer,
 				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel) {
 			super(conProvider, conInfo, bufferTokenName, debugLevel);

From 24c9e0e9a799c99832e019ca0a829c9bc05b1c65 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Jun 2014 14:55:25 +0800
Subject: [PATCH 275/881] enum ConnectionType has been added

We need this to tune the connections.
---
 .../impl/distributed/common/Connection.java   | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 1cd0e11a..4cdfe53c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -195,5 +195,33 @@ public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
 				int timeOut) {
 			throw new java.lang.Error("This method is not supposed to call");
 		}
+
+	}
+
+	/**
+	 * ConnectionType serves two purposes
+	 * <ol>
+	 * <li>Tune the connections. This will passed to opentuner.
+	 * <li>Indicate the {@link StreamNode} to create appropriate
+	 * {@link BoundaryChannel}. This will be bound with {@link ConnectionInfo}.
+	 * </ol>
+	 */
+	public enum ConnectionType {
+		/**
+		 * Blocking TCP socket connection
+		 */
+		BTCP, /**
+		 * Non-Blocking TCP socket connection
+		 */
+		NBTCP, /**
+		 * Asynchronous TCP socket connection
+		 */
+		ATCP, /**
+		 * Blocking InfiniBand
+		 */
+		BIB, /**
+		 * Non-Blocking InfiniBand
+		 */
+		NBIB
 	}
 }

From c557846325f222433baf8e023186eaafd6eacf50 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Jun 2014 12:09:12 +0800
Subject: [PATCH 276/881] ConnectionInfo has been made to an abstract class

ConnectionInfo is made abstract and GenericConnectionInfo is added.
---
 .../impl/distributed/common/Connection.java   | 25 +++++++++++++++++--
 .../impl/distributed/common/Tester.java       | 23 +++++++++++++----
 .../impl/distributed/runtimer/Controller.java |  3 ++-
 3 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 4cdfe53c..919f6c08 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -84,7 +84,7 @@ public interface Connection {
 	 * hashCode() and equals() methods. <b>The whole point of this class is to
 	 * identify a connection between two machines.</b>
 	 */
-	public class ConnectionInfo implements Serializable {
+	public abstract class ConnectionInfo implements Serializable {
 
 		private static final long serialVersionUID = 1L;
 
@@ -191,11 +191,32 @@ public String toString() {
 		 * @return {@link Connection} that is described by this
 		 *         {@link ConnectionInfo}.
 		 */
+		public abstract Connection makeConnection(int nodeID,
+				NetworkInfo networkInfo, int timeOut);
+	}
+
+	/**
+	 * We need an instance of {@link ConnectionInfo} to compare and get a
+	 * concrete {@link ConnectionInfo} from the list of already created
+	 * {@link ConnectionInfo}s. This class is added for that purpose.
+	 */
+	public static class GenericConnectionInfo extends ConnectionInfo {
+
+		private static final long serialVersionUID = 1L;
+
+		public GenericConnectionInfo(int srcID, int dstID) {
+			super(srcID, dstID);
+		}
+
+		public GenericConnectionInfo(int srcID, int dstID, boolean isSymmetric) {
+			super(srcID, dstID, isSymmetric);
+		}
+
+		@Override
 		public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
 				int timeOut) {
 			throw new java.lang.Error("This method is not supposed to call");
 		}
-
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Tester.java b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
index 1319a911..86b746ba 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Tester.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
@@ -7,13 +7,19 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
+;
+
 public class Tester {
 
 	/**
@@ -21,7 +27,8 @@ public class Tester {
 	 */
 	public static void main(String[] args) {
 		// test1();
-		test2();
+		// test2();
+		test3();
 	}
 
 	/**
@@ -71,10 +78,10 @@ private static void test2() {
 		ConnectionInfo tcpConInfo3 = new TCPConnectionInfo(4, 1, 8989);
 		ConnectionInfo tcpConInfo4 = new TCPConnectionInfo(4, 1, 8980);
 
-		ConnectionInfo conInfo1 = new ConnectionInfo(1, 4, true);
-		ConnectionInfo conInfo2 = new ConnectionInfo(1, 4, false);
-		ConnectionInfo conInfo3 = new ConnectionInfo(4, 1, true);
-		ConnectionInfo conInfo4 = new ConnectionInfo(4, 1, false);
+		ConnectionInfo conInfo1 = new GenericConnectionInfo(1, 4, true);
+		ConnectionInfo conInfo2 = new GenericConnectionInfo(1, 4, false);
+		ConnectionInfo conInfo3 = new GenericConnectionInfo(4, 1, true);
+		ConnectionInfo conInfo4 = new GenericConnectionInfo(4, 1, false);
 
 		System.out.println("AsyncTCPConnectionInfo - AsyncTCPConnectionInfo");
 		System.out.println(asyConInfo1.equals(asyConInfo2));
@@ -144,4 +151,10 @@ private static void test2() {
 		System.out.println(tesMap.containsKey(conInfo4));
 	}
 
+	private static void test3() {
+		List<ConnectionType> conlist = Arrays.asList(ConnectionType.values());
+		for (ConnectionType connectionType : conlist) {
+			System.out.println(connectionType);
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index d3ef0656..31e21100 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
@@ -194,7 +195,7 @@ private void addtoconInfoMap(int srcID, int dstID, Token t,
 			Set<ConnectionInfo> usedConInfos,
 			Map<Token, ConnectionInfo> conInfoMap) {
 
-		ConnectionInfo conInfo = new ConnectionInfo(srcID, dstID);
+		ConnectionInfo conInfo = new GenericConnectionInfo(srcID, dstID);
 
 		List<ConnectionInfo> conSet = getTcpConInfo(conInfo);
 		ConnectionInfo tcpConInfo = null;

From f65ebb40fe723cd737c7fd1c9d7353ea7ac4f890 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Jun 2014 13:58:20 +0800
Subject: [PATCH 277/881] ConnectionInfo creates Boundary Channels.

ConnectionInfo creates appropriate connection as well as appropriate
BoundaryChannels.
---
 .../common/AsynchronousTCPConnection.java     | 21 +++++++++++++++++-
 .../impl/distributed/common/Connection.java   | 22 +++++++++++++++++++
 .../distributed/common/TCPConnection.java     | 16 +++++++++++++-
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index a311d3b8..e80cb01d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -14,6 +14,10 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import edu.mit.streamjit.impl.blob.AbstractWriteOnlyBuffer;
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -159,7 +163,8 @@ public <T> T readObject() throws IOException, ClassNotFoundException {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!bBAos.newWrite());
+		while (!bBAos.newWrite())
+			;
 		this.ooStream.write('\u001a');
 		this.ooStream.flush();
 		bBAos.writeCompleted();
@@ -747,6 +752,20 @@ else if (dstID == nodeID) {
 			}
 			return con;
 		}
+
+		@Override
+		public BoundaryInputChannel inputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			// TODO Auto-generated method stub
+			return null;
+		}
+
+		@Override
+		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			// TODO Auto-generated method stub
+			return null;
+		}
 	}
 
 	public static class AsyncTCPBuffer extends AbstractWriteOnlyBuffer {
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 919f6c08..55da7b8e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -4,6 +4,10 @@
 import java.io.ObjectInputStream;
 import java.io.Serializable;
 
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -193,6 +197,12 @@ public String toString() {
 		 */
 		public abstract Connection makeConnection(int nodeID,
 				NetworkInfo networkInfo, int timeOut);
+
+		public abstract BoundaryInputChannel inputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider);
+
+		public abstract BoundaryOutputChannel outputChannel(Token t,
+				int bufSize, TCPConnectionProvider conProvider);
 	}
 
 	/**
@@ -217,6 +227,18 @@ public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
 				int timeOut) {
 			throw new java.lang.Error("This method is not supposed to call");
 		}
+
+		@Override
+		public BoundaryInputChannel inputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			throw new java.lang.Error("This method is not supposed to call");
+		}
+
+		@Override
+		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			throw new java.lang.Error("This method is not supposed to call");
+		}
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 76843340..1dd2eea1 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -2,12 +2,14 @@
 
 import java.io.*;
 import java.net.*;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import static com.google.common.base.Preconditions.*;
 
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -244,6 +246,18 @@ else if (dstID == nodeID) {
 			}
 			return con;
 		}
+
+		@Override
+		public BoundaryInputChannel inputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			throw new java.lang.Error("This method is not supposed to call");
+		}
+
+		@Override
+		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
+				TCPConnectionProvider conProvider) {
+			throw new java.lang.Error("This method is not supposed to call");
+		}
 	}
 
 	/**

From 0eff8f694302a76ea37819617e489f1eccae955b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Jun 2014 14:40:41 +0800
Subject: [PATCH 278/881] abstract methods implemented.

inputChannel() and outputChannel() have been implemented.
---
 .../distributed/common/AsynchronousTCPConnection.java    | 9 +++++----
 .../streamjit/impl/distributed/common/TCPConnection.java | 8 ++++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index e80cb01d..9f6de7cb 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -18,7 +18,9 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
 
 /**
  * Uses {@link AsynchronousSocketChannel} from Java's NIO.2 to send data. This
@@ -756,15 +758,14 @@ else if (dstID == nodeID) {
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			// TODO Auto-generated method stub
-			return null;
+			return new TCPInputChannel(bufSize, conProvider, this,
+					t.toString(), 0);
 		}
 
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			// TODO Auto-generated method stub
-			return null;
+			return new AsyncTCPOutputChannel(conProvider, this, t.toString(), 0);
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 1dd2eea1..85e7619e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -11,6 +11,8 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
 
 /**
  * TCPConnection is not thread safe.
@@ -250,13 +252,15 @@ else if (dstID == nodeID) {
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			throw new java.lang.Error("This method is not supposed to call");
+			return new TCPInputChannel(bufSize, conProvider, this,
+					t.toString(), 0);
 		}
 
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			throw new java.lang.Error("This method is not supposed to call");
+			return new TCPOutputChannel(bufSize, conProvider, this,
+					t.toString(), 0);
 		}
 	}
 

From 799afa161e0090b7c0afe077834f2b5bb2bd4638 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Jun 2014 13:21:38 +0800
Subject: [PATCH 279/881] Uses mixture of Async and blocking connections

StreamNode no need to worry about the connections type. Controller decides
connection type and send the connection info to the stream node. ConnectionInfo
makes appropriate connection, appropriate input channel and appropriate output
channel.
---
 .../impl/distributed/StreamJitAppManager.java |  15 ++-
 .../common/BoundaryChannelManager.java        | 117 ++++++++++++++++++
 .../distributed/node/BlobsManagerImpl.java    |  20 ++-
 3 files changed, 137 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 174dae12..5a80b0b0 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -18,6 +18,7 @@
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Command;
@@ -36,6 +37,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 public class StreamJitAppManager {
@@ -170,9 +172,16 @@ private void setupHeadTail(Map<Token, ConnectionInfo> conInfoMap,
 			throw new IllegalArgumentException(
 					"No head buffer in the passed bufferMap.");
 
-		headChannel = new HeadChannel.TCPHeadChannel(bufferMap.get(headToken),
-				controller.getConProvider(), headconInfo, "headChannel - "
-						+ headToken.toString(), 0);
+		if (headconInfo instanceof TCPConnectionInfo)
+			headChannel = new HeadChannel.TCPHeadChannel(
+					bufferMap.get(headToken), controller.getConProvider(),
+					headconInfo, "headChannel - " + headToken.toString(), 0);
+		else if (headconInfo instanceof AsyncTCPConnectionInfo)
+			headChannel = new HeadChannel.AsyncTCPHeadChannel(
+					bufferMap.get(headToken), controller.getConProvider(),
+					headconInfo, "headChannel - " + headToken.toString(), 0);
+		else
+			throw new IllegalStateException("Head ConnectionInfo doesn't match");
 
 		ConnectionInfo tailconInfo = conInfoMap.get(tailToken);
 		assert tailconInfo != null : "No tail connection info exists in conInfoMap";
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
index 67b212ca..e604222b 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
@@ -1,9 +1,15 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.*;
+import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 
 /**
  * Manages set of {@link BoundaryChannel}s.
@@ -54,4 +60,115 @@ public interface BoundaryOutputChannelManager
 		 */
 		void stop(boolean stopType);
 	}
+
+	public static class InputChannelManager
+			implements
+				BoundaryInputChannelManager {
+
+		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
+
+		private final Set<Thread> inputChannelThreads;
+
+		public InputChannelManager(
+				final ImmutableMap<Token, BoundaryInputChannel> inputChannels) {
+			this.inputChannels = inputChannels;
+			inputChannelThreads = new HashSet<>(inputChannels.values().size());
+		}
+
+		@Override
+		public void start() {
+			for (BoundaryInputChannel bc : inputChannels.values()) {
+				Thread t = new Thread(bc.getRunnable(), bc.name());
+				t.start();
+				inputChannelThreads.add(t);
+			}
+		}
+
+		@Override
+		public void waitToStart() {
+		}
+
+		@Override
+		public void waitToStop() {
+			for (Thread t : inputChannelThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		@Override
+		public void stop(int stopType) {
+			for (BoundaryInputChannel bc : inputChannels.values()) {
+				bc.stop(stopType);
+			}
+		}
+
+		@Override
+		public ImmutableMap<Token, BoundaryInputChannel> inputChannelsMap() {
+			return inputChannels;
+		}
+	}
+
+	public static class OutputChannelManager
+			implements
+				BoundaryOutputChannelManager {
+
+		protected final ImmutableMap<Token, BoundaryOutputChannel> outputChannels;
+		protected final Map<BoundaryOutputChannel, Thread> outputChannelThreads;
+
+		public OutputChannelManager(
+				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
+			this.outputChannels = outputChannels;
+			outputChannelThreads = new HashMap<>(outputChannels.values().size());
+		}
+
+		@Override
+		public void start() {
+			for (BoundaryOutputChannel bc : outputChannels.values()) {
+				Thread t = new Thread(bc.getRunnable(), bc.name());
+				t.start();
+				outputChannelThreads.put(bc, t);
+			}
+		}
+
+		@Override
+		public void waitToStart() {
+			for (Map.Entry<BoundaryOutputChannel, Thread> en : outputChannelThreads
+					.entrySet()) {
+				if (en.getKey() instanceof AsyncTCPOutputChannel) {
+					try {
+						en.getValue().join();
+					} catch (InterruptedException e) {
+						e.printStackTrace();
+					}
+				}
+			}
+		}
+
+		@Override
+		public void stop(boolean stopType) {
+			for (BoundaryOutputChannel bc : outputChannels.values()) {
+				bc.stop(stopType);
+			}
+		}
+
+		@Override
+		public void waitToStop() {
+			for (Thread t : outputChannelThreads.values()) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		@Override
+		public ImmutableMap<Token, BoundaryOutputChannel> outputChannelsMap() {
+			return outputChannels;
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 86781a28..f0e0f4d3 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -25,8 +25,6 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelFactory.*;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.*;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
@@ -54,7 +52,6 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private Map<Token, BlobExecuter> blobExecuters;
 	private final StreamNode streamNode;
-	private final BoundaryChannelFactory chnlFactory;
 	private Map<Token, ConnectionInfo> conInfoMap;
 
 	private MonitorBuffers monBufs;
@@ -67,6 +64,8 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final BufferManager bufferManager;
 
+	private final TCPConnectionProvider conProvider;
+
 	/**
 	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
 	 * time dead lock. Otherwise dynamic buffer will be used for local buffers
@@ -79,7 +78,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			TCPConnectionProvider conProvider) {
 		this.conInfoMap = conInfoMap;
 		this.streamNode = streamNode;
-		this.chnlFactory = new TCPBoundaryChannelFactory(conProvider);
+		this.conProvider = conProvider;
 
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
@@ -149,7 +148,7 @@ private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 		for (Token t : inputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
 			inputChannelMap.put(t,
-					chnlFactory.makeInputChannel(t, bufferMap.get(t), conInfo));
+					conInfo.inputChannel(t, bufferMap.get(t), conProvider));
 		}
 		return inputChannelMap.build();
 	}
@@ -159,9 +158,8 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 		ImmutableMap.Builder<Token, BoundaryOutputChannel> outputChannelMap = new ImmutableMap.Builder<>();
 		for (Token t : outputTokens) {
 			ConnectionInfo conInfo = conInfoMap.get(t);
-			outputChannelMap
-					.put(t, chnlFactory.makeOutputChannel(t, bufferMap.get(t),
-							conInfo));
+			outputChannelMap.put(t,
+					conInfo.outputChannel(t, bufferMap.get(t), conProvider));
 		}
 		return outputChannelMap.build();
 	}
@@ -196,10 +194,8 @@ private BlobExecuter(Token t, Blob blob,
 			this.blobThreads = new HashSet<>();
 			assert blob.getInputs().containsAll(inputChannels.keySet());
 			assert blob.getOutputs().containsAll(outputChannels.keySet());
-			this.inChnlManager = new ChannelManagers.BlockingInputChannelManager(
-					inputChannels);
-			this.outChnlManager = new ChannelManagers.BlockingOutputChannelManager(
-					outputChannels);
+			this.inChnlManager = new InputChannelManager(inputChannels);
+			this.outChnlManager = new OutputChannelManager(outputChannels);
 
 			String baseName = getName(blob);
 			for (int i = 0; i < blob.getCoreCount(); i++) {

From 48c2356350db94be7f91be7b16bf2f5b6bf45897 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Jun 2014 23:31:31 +0800
Subject: [PATCH 280/881] ConnectionManager has been added

Connection managing related methods and codes have been moved from controller
to connection manager.
---
 .../impl/distributed/ConnectionManager.java   | 285 ++++++++++++++++++
 .../impl/distributed/runtimer/Controller.java | 138 ---------
 2 files changed, 285 insertions(+), 138 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/ConnectionManager.java

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
new file mode 100644
index 00000000..ffb0ad24
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -0,0 +1,285 @@
+package edu.mit.streamjit.impl.distributed;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.Builder;
+import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+import edu.mit.streamjit.impl.distributed.common.Connection;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
+import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+
+/**
+ * Generates configuration parameters to tune the {@link Connection}'s
+ * communication type such as blocking TCP connection, asynchronous TCP
+ * connection, Infiniband, etc.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Jun 23, 2014
+ * 
+ */
+public interface ConnectionManager {
+
+	/**
+	 * Generates parameters to tune {@link BoundaryChannel} and add those into
+	 * the {@link Configuration.Builder}.
+	 * 
+	 * @param cfgBuilder
+	 * @param workers
+	 */
+	public void addChannelParameters(Configuration.Builder cfgBuilder,
+			Set<Worker<?, ?>> workers);
+
+	/**
+	 * Generates parameters to tune {@link BoundaryChannel} and return those as
+	 * a new {@link Configuration}.
+	 * 
+	 * @param workers
+	 * @return
+	 */
+	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers);
+
+	/**
+	 * Decides {@link Connection} type for each inter-blob connection based on
+	 * the {@link cfg}.
+	 * 
+	 * @param cfg
+	 * @param partitionsMachineMap
+	 * @param source
+	 * @param sink
+	 * @return
+	 */
+	public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
+			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap,
+			Worker<?, ?> source, Worker<?, ?> sink);
+
+	public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo);
+
+	public abstract static class AbstractConnectionManager
+			implements
+				ConnectionManager {
+
+		private final int controllerNodeID;
+
+		protected Set<ConnectionInfo> currentConInfos;
+
+		protected int startPortNo = 24896; // Just a random magic number.
+
+		public AbstractConnectionManager(int controllerNodeID) {
+			this.controllerNodeID = controllerNodeID;
+			this.currentConInfos = new HashSet<>();
+		}
+
+		public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
+				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap,
+				Worker<?, ?> source, Worker<?, ?> sink) {
+
+			assert partitionsMachineMap != null : "partitionsMachineMap is null";
+
+			Set<ConnectionInfo> usedConInfos = new HashSet<>();
+			Map<Token, ConnectionInfo> conInfoMap = new HashMap<>();
+
+			for (Integer machineID : partitionsMachineMap.keySet()) {
+				List<Set<Worker<?, ?>>> blobList = partitionsMachineMap
+						.get(machineID);
+				Set<Worker<?, ?>> allWorkers = new HashSet<>(); // Contains all
+																// workers those
+																// are
+																// assigned to
+																// the
+																// current
+																// machineID
+																// machine.
+				for (Set<Worker<?, ?>> blobWorkers : blobList) {
+					allWorkers.addAll(blobWorkers);
+				}
+
+				for (Worker<?, ?> w : allWorkers) {
+					for (Worker<?, ?> succ : Workers.getSuccessors(w)) {
+						if (allWorkers.contains(succ))
+							continue;
+						int dstMachineID = getAssignedMachine(succ,
+								partitionsMachineMap);
+						Token t = new Token(w, succ);
+						addtoconInfoMap(machineID, dstMachineID, t,
+								usedConInfos, conInfoMap);
+					}
+				}
+			}
+
+			Token headToken = Token.createOverallInputToken(source);
+			int dstMachineID = getAssignedMachine(source, partitionsMachineMap);
+			addtoconInfoMap(controllerNodeID, dstMachineID, headToken,
+					usedConInfos, conInfoMap);
+
+			Token tailToken = Token.createOverallOutputToken(sink);
+			int srcMahineID = getAssignedMachine(sink, partitionsMachineMap);
+			addtoconInfoMap(srcMahineID, controllerNodeID, tailToken,
+					usedConInfos, conInfoMap);
+
+			return conInfoMap;
+		}
+
+		/**
+		 * @param worker
+		 * @return the machineID where on which the passed worker is assigned.
+		 */
+		private int getAssignedMachine(Worker<?, ?> worker,
+				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+			for (Integer machineID : partitionsMachineMap.keySet()) {
+				for (Set<Worker<?, ?>> workers : partitionsMachineMap
+						.get(machineID)) {
+					if (workers.contains(worker))
+						return machineID;
+				}
+			}
+
+			throw new IllegalArgumentException(String.format(
+					"%s is not assigned to anyof the machines", worker));
+		}
+
+		protected abstract void addtoconInfoMap(int srcID, int dstID, Token t,
+				Set<ConnectionInfo> usedConInfos,
+				Map<Token, ConnectionInfo> conInfoMap);
+
+		protected List<ConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
+			List<ConnectionInfo> conList = new ArrayList<>();
+			for (ConnectionInfo tcpconInfo : currentConInfos) {
+				if (conInfo.equals(tcpconInfo))
+					conList.add(tcpconInfo);
+			}
+			return conList;
+		}
+
+		public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
+			if (currentConInfos.contains(conInfo))
+				currentConInfos.remove(conInfo);
+			ConnectionInfo newConinfo;
+			if (conInfo.getSrcID() == 0)
+				newConinfo = new TCPConnectionInfo(conInfo.getSrcID(),
+						conInfo.getDstID(), startPortNo++);
+			else
+				newConinfo = new AsyncTCPConnectionInfo(conInfo.getSrcID(),
+						conInfo.getDstID(), startPortNo++);
+			currentConInfos.add(newConinfo);
+
+			return newConinfo;
+		}
+	}
+
+	public static class NoConnectionParams extends AbstractConnectionManager {
+
+		public NoConnectionParams(int controllerNodeID) {
+			super(controllerNodeID);
+		}
+
+		@Override
+		public void addChannelParameters(Builder cfgBuilder,
+				Set<Worker<?, ?>> workers) {
+			return;
+		}
+
+		@Override
+		public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
+			return Configuration.builder().build();
+		}
+
+		/**
+		 * Just extracted from {@link #buildConInfoMap(Map, Worker, Worker)}
+		 * because the code snippet in this method happened to repeat three
+		 * times inside the {@link #buildConInfoMap(Map, Worker, Worker)}
+		 * method.
+		 */
+		protected void addtoconInfoMap(int srcID, int dstID, Token t,
+				Set<ConnectionInfo> usedConInfos,
+				Map<Token, ConnectionInfo> conInfoMap) {
+
+			ConnectionInfo conInfo = new GenericConnectionInfo(srcID, dstID);
+
+			List<ConnectionInfo> conSet = getTcpConInfo(conInfo);
+			ConnectionInfo tcpConInfo = null;
+
+			for (ConnectionInfo con : conSet) {
+				if (!usedConInfos.contains(con)) {
+					tcpConInfo = con;
+					break;
+				}
+			}
+
+			if (tcpConInfo == null) {
+				tcpConInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
+				this.currentConInfos.add(tcpConInfo);
+			}
+
+			conInfoMap.put(t, tcpConInfo);
+			usedConInfos.add(tcpConInfo);
+		}
+	}
+
+	public static class AllConnectionParams extends AbstractConnectionManager {
+		public AllConnectionParams(int controllerNodeID) {
+			super(controllerNodeID);
+			// TODO Auto-generated constructor stub
+		}
+
+		@Override
+		public void addChannelParameters(Builder cfgBuilder,
+				Set<Worker<?, ?>> workers) {
+			for (Worker<?, ?> w : workers) {
+				for (Worker<?, ?> succ : Workers.getSuccessors(w)) {
+					Token t = new Token(w, succ);
+					Parameter p = new Configuration.SwitchParameter<ConnectionType>(
+							getParamName(t), ConnectionType.class,
+							ConnectionType.BTCP, Arrays.asList(ConnectionType
+									.values()));
+					cfgBuilder.addParameter(p);
+				}
+			}
+
+			// Add Parameter for global input channel.
+			Set<Worker<?, ?>> heads = Workers.getTopmostWorkers(workers);
+			assert heads.size() == 1 : "Multiple first workers";
+			for (Worker<?, ?> firstWorker : heads) {
+				Token t = Token.createOverallInputToken(firstWorker);
+				Parameter p = new Configuration.SwitchParameter<ConnectionType>(
+						getParamName(t), ConnectionType.class,
+						ConnectionType.BTCP, Arrays.asList(ConnectionType
+								.values()));
+				cfgBuilder.addParameter(p);
+			}
+		}
+
+		@Override
+		public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
+			Configuration.Builder cfgBuilder = Configuration.builder();
+			addChannelParameters(cfgBuilder, workers);
+			return cfgBuilder.build();
+		}
+
+		protected String getParamName(Token t) {
+			return String.format("ConnectionType-%d:%d",
+					t.getUpstreamIdentifier(), t.getDownstreamIdentifier());
+		}
+
+		@Override
+		protected void addtoconInfoMap(int srcID, int dstID, Token t,
+				Set<ConnectionInfo> usedConInfos,
+				Map<Token, ConnectionInfo> conInfoMap) {
+			// TODO Auto-generated method stub
+
+		}
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 31e21100..b494ed6a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -2,29 +2,19 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
-import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
@@ -44,8 +34,6 @@ public class Controller {
 
 	private TCPConnectionProvider conProvider;
 
-	private int startPortNo = 24896; // Just a random magic number.
-
 	private CommunicationManager comManager;
 
 	/**
@@ -61,12 +49,9 @@ public class Controller {
 	 */
 	public final int controllerNodeID;
 
-	private Set<ConnectionInfo> currentConInfos;
-
 	public Controller() {
 		this.comManager = new BlockingCommunicationManager();
 		this.controllerNodeID = 0;
-		this.currentConInfos = new HashSet<>();
 	}
 
 	/**
@@ -139,96 +124,6 @@ public void newApp(Configuration staticCfg) {
 		sendToAll(json);
 	}
 
-	public Map<Token, ConnectionInfo> buildConInfoMap(
-			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap,
-			Worker<?, ?> source, Worker<?, ?> sink) {
-
-		assert partitionsMachineMap != null : "partitionsMachineMap is null";
-
-		Set<ConnectionInfo> usedConInfos = new HashSet<>();
-		Map<Token, ConnectionInfo> conInfoMap = new HashMap<>();
-
-		for (Integer machineID : partitionsMachineMap.keySet()) {
-			List<Set<Worker<?, ?>>> blobList = partitionsMachineMap
-					.get(machineID);
-			Set<Worker<?, ?>> allWorkers = new HashSet<>(); // Contains all
-															// workers those are
-															// assigned to the
-															// current machineID
-															// machine.
-			for (Set<Worker<?, ?>> blobWorkers : blobList) {
-				allWorkers.addAll(blobWorkers);
-			}
-
-			for (Worker<?, ?> w : allWorkers) {
-				for (Worker<?, ?> succ : Workers.getSuccessors(w)) {
-					if (allWorkers.contains(succ))
-						continue;
-					int dstMachineID = getAssignedMachine(succ,
-							partitionsMachineMap);
-					Token t = new Token(w, succ);
-					addtoconInfoMap(machineID, dstMachineID, t, usedConInfos,
-							conInfoMap);
-				}
-			}
-		}
-
-		Token headToken = Token.createOverallInputToken(source);
-		int dstMachineID = getAssignedMachine(source, partitionsMachineMap);
-		addtoconInfoMap(controllerNodeID, dstMachineID, headToken,
-				usedConInfos, conInfoMap);
-
-		Token tailToken = Token.createOverallOutputToken(sink);
-		int srcMahineID = getAssignedMachine(sink, partitionsMachineMap);
-		addtoconInfoMap(srcMahineID, controllerNodeID, tailToken, usedConInfos,
-				conInfoMap);
-
-		return conInfoMap;
-	}
-
-	/**
-	 * Just extracted from {@link #buildConInfoMap(Map, Worker, Worker)} because
-	 * the code snippet in this method happened to repeat three times inside the
-	 * {@link #buildConInfoMap(Map, Worker, Worker)} method.
-	 */
-	private void addtoconInfoMap(int srcID, int dstID, Token t,
-			Set<ConnectionInfo> usedConInfos,
-			Map<Token, ConnectionInfo> conInfoMap) {
-
-		ConnectionInfo conInfo = new GenericConnectionInfo(srcID, dstID);
-
-		List<ConnectionInfo> conSet = getTcpConInfo(conInfo);
-		ConnectionInfo tcpConInfo = null;
-
-		for (ConnectionInfo con : conSet) {
-			if (!usedConInfos.contains(con)) {
-				tcpConInfo = con;
-				break;
-			}
-		}
-
-		if (tcpConInfo == null) {
-			if (srcID == 0)
-				tcpConInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
-			else
-				tcpConInfo = new AsyncTCPConnectionInfo(srcID, dstID,
-						startPortNo++);
-			this.currentConInfos.add(tcpConInfo);
-		}
-
-		conInfoMap.put(t, tcpConInfo);
-		usedConInfos.add(tcpConInfo);
-	}
-
-	private List<ConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
-		List<ConnectionInfo> conList = new ArrayList<>();
-		for (ConnectionInfo tcpconInfo : currentConInfos) {
-			if (conInfo.equals(tcpconInfo))
-				conList.add(tcpconInfo);
-		}
-		return conList;
-	}
-
 	public Set<Integer> getAllNodeIDs() {
 		return StreamNodeMap.keySet();
 	}
@@ -244,24 +139,6 @@ public void send(int nodeID, CTRLRMessageElement message) {
 		}
 	}
 
-	/**
-	 * @param worker
-	 * @return the machineID where on which the passed worker is assigned.
-	 */
-	private int getAssignedMachine(Worker<?, ?> worker,
-			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
-		for (Integer machineID : partitionsMachineMap.keySet()) {
-			for (Set<Worker<?, ?>> workers : partitionsMachineMap
-					.get(machineID)) {
-				if (workers.contains(worker))
-					return machineID;
-			}
-		}
-
-		throw new IllegalArgumentException(String.format(
-				"%s is not assigned to anyof the machines", worker));
-	}
-
 	public void sendToAll(Object object) {
 		for (StreamNodeAgent node : StreamNodeMap.values()) {
 			try {
@@ -289,19 +166,4 @@ public void registerManager(StreamJitAppManager manager) {
 			node.registerManager(manager);
 		}
 	}
-
-	public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
-		if (currentConInfos.contains(conInfo))
-			currentConInfos.remove(conInfo);
-		ConnectionInfo newConinfo;
-		if (conInfo.getSrcID() == 0)
-			newConinfo = new TCPConnectionInfo(conInfo.getSrcID(),
-					conInfo.getDstID(), startPortNo++);
-		else
-			newConinfo = new AsyncTCPConnectionInfo(conInfo.getSrcID(),
-					conInfo.getDstID(), startPortNo++);
-		currentConInfos.add(newConinfo);
-
-		return newConinfo;
-	}
 }

From a19226abcde6304b497f193c24cdbdf29a61dfde Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Jun 2014 23:38:41 +0800
Subject: [PATCH 281/881] Dist compiler&Blobfactory use ConnectionManager

---
 .../distributed/DistributedBlobFactory.java     | 17 ++++++++++++-----
 .../distributed/DistributedStreamCompiler.java  |  8 +++++---
 .../impl/distributed/StreamJitAppManager.java   | 11 +++++++----
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 851d15aa..a713094e 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -9,6 +9,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.NoConnectionParams;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
@@ -25,7 +26,7 @@
  * <p>
  * TODO: For the moment this factory just deal with compiler blob. Need to make
  * interpreter blob as well based on the dynamic edges.
- *
+ * 
  * @author Sumanan sumanan@mit.edu
  * @since Sep 24, 2013
  */
@@ -35,26 +36,30 @@ public class DistributedBlobFactory implements BlobFactory {
 
 	private final ConfigurationManager cfgManager;
 
+	private final ConnectionManager connectionManager;
+
 	public DistributedBlobFactory(ConfigurationManager cfgManager,
-			int noOfMachines) {
+			ConnectionManager connectionManager, int noOfMachines) {
 		this.cfgManager = cfgManager;
 		this.noOfMachines = noOfMachines;
+		this.connectionManager = connectionManager;
 	}
 
 	/**
 	 * If {@link ConfigurationManager} is not passed as a constructor argument
 	 * then {@link WorkerMachine} will be used as default one.
-	 *
+	 * 
 	 * @param noOfMachines
 	 */
 	public DistributedBlobFactory(int noOfMachines) {
-		this(new WorkerMachine(null), noOfMachines);
+		this(new WorkerMachine(null), new NoConnectionParams(0), noOfMachines);
 	}
 
 	@Override
 	public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 			int maxNumCores, DrainData initialState) {
-		return new Compiler2BlobFactory().makeBlob(workers, config, maxNumCores, initialState);
+		return new Compiler2BlobFactory().makeBlob(workers, config,
+				maxNumCores, initialState);
 	}
 
 	@Override
@@ -70,6 +75,8 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
+
+		connectionManager.addChannelParameters(builder, workers);
 		return builder.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 4418fbe6..b47490a2 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -135,8 +135,10 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		StreamJitApp app = new StreamJitApp(stream, source, sink);
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, Math.max(
-				noOfnodes - 1, 1));
+		ConnectionManager conManager = new ConnectionManager.NoConnectionParams(
+				controller.controllerNodeID);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
+				Math.max(noOfnodes - 1, 1));
 		this.cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
@@ -182,7 +184,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			Portals.setConstraints(portal, constraints);
 
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
-				cfgManager);
+				cfgManager, conManager);
 		final AbstractDrainer drainer = new DistributedDrainer(manager);
 		drainer.setBlobGraph(app.blobGraph);
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 5a80b0b0..a2b34ecd 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -52,6 +52,8 @@ public class StreamJitAppManager {
 
 	private final Controller controller;
 
+	private final ConnectionManager conManager;
+
 	private final StreamJitApp app;
 
 	private final ConfigurationManager cfgManager;
@@ -87,10 +89,11 @@ public class StreamJitAppManager {
 	private Map<Token, ConnectionInfo> conInfoMap;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
-			ConfigurationManager cfgManager) {
+			ConfigurationManager cfgManager, ConnectionManager conManager) {
 		this.controller = controller;
 		this.app = app;
 		this.cfgManager = cfgManager;
+		this.conManager = conManager;
 		this.status = AppStatus.NOT_STARTED;
 		this.exP = new SNExceptionProcessorImpl();
 		this.ep = new ErrorProcessorImpl();
@@ -111,8 +114,8 @@ public boolean reconfigure(int multiplier) {
 		Configuration.Builder builder = Configuration.builder(cfgManager
 				.getDynamicConfiguration());
 
-		conInfoMap = controller.buildConInfoMap(app.partitionsMachineMap,
-				app.source, app.sink);
+		conInfoMap = conManager.conInfoMap(app.blobConfiguration,
+				app.partitionsMachineMap, app.source, app.sink);
 
 		builder.putExtraData(GlobalConstants.CONINFOMAP, conInfoMap);
 
@@ -402,7 +405,7 @@ public void process(AddressBindException abEx) {
 							"Illegal TCP connection - " + abEx.conInfo);
 				}
 
-				ConnectionInfo coninfo = controller
+				ConnectionInfo coninfo = conManager
 						.getNewTCPConInfo(abEx.conInfo);
 
 				exConInfos.add(abEx.conInfo);

From 8829cad3c0aaa185a26f58283ad65859dbf9133f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Jun 2014 23:55:06 +0800
Subject: [PATCH 282/881] NoParams connection manager has been added

NoParams add no connection related parameters for tuning.
---
 .../impl/distributed/ConnectionManager.java   | 41 +++++++++++++++----
 .../distributed/DistributedBlobFactory.java   |  4 +-
 .../DistributedStreamCompiler.java            |  2 +-
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index ffb0ad24..d68b4059 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -180,9 +180,9 @@ public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
 		}
 	}
 
-	public static class NoConnectionParams extends AbstractConnectionManager {
+	public static abstract class NoParams extends AbstractConnectionManager {
 
-		public NoConnectionParams(int controllerNodeID) {
+		public NoParams(int controllerNodeID) {
 			super(controllerNodeID);
 		}
 
@@ -197,12 +197,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			return Configuration.builder().build();
 		}
 
-		/**
-		 * Just extracted from {@link #buildConInfoMap(Map, Worker, Worker)}
-		 * because the code snippet in this method happened to repeat three
-		 * times inside the {@link #buildConInfoMap(Map, Worker, Worker)}
-		 * method.
-		 */
 		protected void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,
 				Map<Token, ConnectionInfo> conInfoMap) {
@@ -220,13 +214,42 @@ protected void addtoconInfoMap(int srcID, int dstID, Token t,
 			}
 
 			if (tcpConInfo == null) {
-				tcpConInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
+				tcpConInfo = makeConnectionInfo(srcID, dstID, startPortNo++);
 				this.currentConInfos.add(tcpConInfo);
 			}
 
 			conInfoMap.put(t, tcpConInfo);
 			usedConInfos.add(tcpConInfo);
 		}
+
+		protected abstract ConnectionInfo makeConnectionInfo(int srcID,
+				int dstID, int portNo);
+	}
+
+	public static class BlockingTCPNoParams extends NoParams {
+
+		public BlockingTCPNoParams(int controllerNodeID) {
+			super(controllerNodeID);
+		}
+
+		@Override
+		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID,
+				int portNo) {
+			return new TCPConnectionInfo(srcID, dstID, portNo);
+		}
+	}
+
+	public static class AsyncTCPNoParams extends NoParams {
+
+		public AsyncTCPNoParams(int controllerNodeID) {
+			super(controllerNodeID);
+		}
+
+		@Override
+		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID,
+				int portNo) {
+			return new AsyncTCPConnectionInfo(srcID, dstID, portNo);
+		}
 	}
 
 	public static class AllConnectionParams extends AbstractConnectionManager {
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index a713094e..971b8ea5 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -9,7 +9,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
-import edu.mit.streamjit.impl.distributed.ConnectionManager.NoConnectionParams;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
@@ -52,7 +52,7 @@ public DistributedBlobFactory(ConfigurationManager cfgManager,
 	 * @param noOfMachines
 	 */
 	public DistributedBlobFactory(int noOfMachines) {
-		this(new WorkerMachine(null), new NoConnectionParams(0), noOfMachines);
+		this(new WorkerMachine(null), new BlockingTCPNoParams(0), noOfMachines);
 	}
 
 	@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index b47490a2..b1757f33 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -135,7 +135,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		StreamJitApp app = new StreamJitApp(stream, source, sink);
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
-		ConnectionManager conManager = new ConnectionManager.NoConnectionParams(
+		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));

From a1a6f804a34e4eea5ba3e1ec0ef42d8223f82e11 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 21 Jul 2014 15:19:29 +0800
Subject: [PATCH 283/881] Moved getParamName() from subclass to superclass

---
 .../impl/distributed/ConnectionManager.java       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index d68b4059..acf6f618 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -68,9 +68,8 @@ public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
 
 	public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo);
 
-	public abstract static class AbstractConnectionManager
-			implements
-				ConnectionManager {
+	public abstract static class AbstractConnectionManager implements
+			ConnectionManager {
 
 		private final int controllerNodeID;
 
@@ -164,6 +163,11 @@ protected List<ConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
 			return conList;
 		}
 
+		protected String getParamName(Token t) {
+			return String.format("ConnectionType-%d:%d",
+					t.getUpstreamIdentifier(), t.getDownstreamIdentifier());
+		}
+
 		public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
 			if (currentConInfos.contains(conInfo))
 				currentConInfos.remove(conInfo);
@@ -292,11 +296,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			return cfgBuilder.build();
 		}
 
-		protected String getParamName(Token t) {
-			return String.format("ConnectionType-%d:%d",
-					t.getUpstreamIdentifier(), t.getDownstreamIdentifier());
-		}
-
 		@Override
 		protected void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,

From ecb5f0ea87acab0f98a8524abf671e89f967652d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 13 Aug 2014 12:08:41 +0800
Subject: [PATCH 284/881] Bug fix : NullPointerException.

AsyncTCPOutputChannel.stop() get called before connection is created. This
throws NullPointerException. This event happens at the start of the tuning
process. i.e., Before HeadChannel is created, OnlineTuner.reconfigure() calls
drainer.startDraining(0).
---
 .../impl/distributed/node/AsyncTCPOutputChannel.java          | 4 +++-
 .../mit/streamjit/impl/distributed/node/TCPInputChannel.java  | 1 +
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 6769ef7f..3cf522ea 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -14,7 +14,7 @@
 
 public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
-	private Connection con;
+	private volatile Connection con;
 
 	private final String name;
 
@@ -67,6 +67,8 @@ public ImmutableList<Object> getUnprocessedData() {
 
 	@Override
 	public void stop(boolean isFinal) {
+		while (con == null)
+			;
 		this.isFinal = isFinal;
 		if (!stopCalled) {
 			try {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index cdd0f9bc..0438e4f7 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -291,6 +291,7 @@ private void finalReceive() {
 				softClosed = true;
 				hasData = false;
 			} catch (IOException e) {
+				e.printStackTrace();
 				System.out
 						.println("finalReceive:Closing by IOException. Not by softClose.");
 				hasData = false;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f240bfb3..02a37e3d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -182,6 +182,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				time = -1l;
 			}
 		} catch (Exception ex) {
+			ex.printStackTrace();
 			System.err
 					.println("Couldn't compile the stream graph with this configuration");
 			time = -1l;

From 5e04d060f3bcf128f414adf007735edc51ca5abd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 09:25:22 +0800
Subject: [PATCH 285/881] filewriter has been made as private final.

---
 .../mit/streamjit/impl/distributed/node/TCPOutputChannel.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index a6a9184a..1d0b38d5 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -28,7 +28,7 @@
  */
 public class TCPOutputChannel implements BoundaryOutputChannel {
 
-	FileWriter writer;
+	private final FileWriter writer;
 
 	private final int debugLevel;
 

From a29a090713aaebd4a04ebc1d70c4f8255b2bd8ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 09:47:32 +0800
Subject: [PATCH 286/881] Throws exception with discriptive exp message.

WriteCompleted() throws buffer state conflict exception with discriptive
exception messaged.
---
 .../distributed/common/AsynchronousTCPConnection.java    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 9f6de7cb..72589fbb 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -617,8 +617,13 @@ public void writeCompleted() {
 			writeIndex = (writeIndex + 1) % bytebufferArray.length;
 			boolean ret = bufferStatus.get(w).compareAndSet(
 					Status.beingWritten, Status.canRead);
-			if (!ret)
-				throw new IllegalStateException("bufferStatus conflict");
+			if (!ret) {
+				String msg = String.format("BufferState conflict : " + "writeIndex - "
+						+ writeIndex + ", readIndex - " + readIndex
+						+ " - Status of the writeBuffer is "
+						+ bufferStatus.get(w).get());
+				throw new IllegalStateException(msg);
+			}
 		}
 
 		/**

From e9bba6275c3c5ce294cabe0aa0ecbc0aa0bb66d8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 12:39:44 +0800
Subject: [PATCH 287/881] Variable renaming : debugPrint -> debugLevel.

---
 .../common/AsynchronousTCPConnection.java      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index 72589fbb..a37e0c31 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -538,7 +538,7 @@ private enum Status {
 	 */
 	public class ByteBufferArrayOutputStream extends OutputStream {
 
-		private final int debugPrint;
+		private final int debugLevel;
 
 		/**
 		 * Read index of {@link #bytebufferArray}.
@@ -559,7 +559,7 @@ public class ByteBufferArrayOutputStream extends OutputStream {
 		private Map<Integer, AtomicReference<Status>> bufferStatus;
 
 		public ByteBufferArrayOutputStream(int listSize) {
-			debugPrint = 0;
+			debugLevel = 0;
 			writeIndex = 0;
 			readIndex = 0;
 			bytebufferArray = new ByteBufferOutputStream[listSize];
@@ -590,13 +590,13 @@ public void write(byte b[], int off, int len) throws IOException {
 		public boolean newWrite() {
 			if (bufferStatus.get(writeIndex).compareAndSet(Status.canWrite,
 					Status.beingWritten)) {
-				if (debugPrint > 0)
+				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newWrite-canWrite : " + "writeIndex - "
 							+ writeIndex + ", readIndex - " + readIndex);
 				return true;
 			} else {
-				if (debugPrint > 0)
+				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newWrite-failed : " + "writeIndex - "
 							+ writeIndex + ", readIndex - " + readIndex);
@@ -609,7 +609,7 @@ public boolean newWrite() {
 		 * of objects is completed.
 		 */
 		public void writeCompleted() {
-			if (debugPrint > 0)
+			if (debugLevel > 0)
 				System.out.println(Thread.currentThread().getName()
 						+ " : writeCompleted : " + "writeIndex - " + writeIndex
 						+ ", readIndex - " + readIndex);
@@ -636,7 +636,7 @@ public void writeCompleted() {
 		 */
 		public synchronized ByteBufferOutputStream newRead() {
 			if (bufferStatus.get(readIndex).get() == Status.beingRead) {
-				if (debugPrint > 0)
+				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newRead-beingRead : " + "writeIndex - "
 							+ writeIndex + ", readIndex - " + readIndex);
@@ -645,7 +645,7 @@ public synchronized ByteBufferOutputStream newRead() {
 
 			if (bufferStatus.get(readIndex).compareAndSet(Status.canRead,
 					Status.beingRead)) {
-				if (debugPrint > 0)
+				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newRead-canRead : " + "writeIndex - "
 							+ writeIndex + ", readIndex - " + readIndex);
@@ -655,7 +655,7 @@ public synchronized ByteBufferOutputStream newRead() {
 				}
 				return bytebufferArray[readIndex];
 			} else {
-				if (debugPrint > 0)
+				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newRead - not can read " + readIndex);
 				return null;
@@ -667,7 +667,7 @@ public synchronized ByteBufferOutputStream newRead() {
 		 * completed.
 		 */
 		public void readCompleted() {
-			if (debugPrint > 0)
+			if (debugLevel > 0)
 				System.out.println(Thread.currentThread().getName()
 						+ " : readCompleted : " + "writeIndex - " + writeIndex
 						+ ", readIndex - " + readIndex);

From cd83ab8923c3cdfafc62fb152745010b77e85f54 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 21:12:32 +0800
Subject: [PATCH 288/881] BugFix:Sends all data b4 reading for next round

Ensures that data read are sent before going for next round of reading from
the readBuffer.
---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 043438f5..b0e52ab4 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -35,6 +35,7 @@ public static class AsyncTCPHeadChannel extends AsyncTCPOutputChannel {
 
 		final Buffer readBuffer;
 		private volatile boolean stopCalled;
+
 		public AsyncTCPHeadChannel(Buffer buffer,
 				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel) {
@@ -54,13 +55,19 @@ public void run() {
 					final int dataLength = 10000;
 					final Object[] data = new Object[dataLength];
 					int read = 1;
+					int written = 0;
 					while (read != 0 && !stopCalled) {
 						read = readBuffer.read(data, 0, data.length);
-						writeBuffer.write(data, 0, read);
+						written = 0;
+						while (written < read) {
+							written += writeBuffer.write(data, written, read
+									- written);
+						}
 					}
 				}
 			};
 		}
+
 		protected void fillUnprocessedData() {
 			throw new Error("Method not implemented");
 		}

From 7141bf199c5fa3fd722b7058ef3a159f6d5f022e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 21:20:42 +0800
Subject: [PATCH 289/881] Thread sleeps if writing failed.

Writing failed means all byteBuffers are full and java threads responsible for
Asynchronous socket are busy with sending data over network. So it is
appropriate to sleep.
---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index b0e52ab4..c58ea363 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -62,6 +62,14 @@ public void run() {
 						while (written < read) {
 							written += writeBuffer.write(data, written, read
 									- written);
+							if (written == 0) {
+								try {
+									// TODO: Verify this sleep time.
+									Thread.sleep(500);
+								} catch (InterruptedException e) {
+									e.printStackTrace();
+								}
+							}
 						}
 					}
 				}

From d6cab30d415703d57920a25f442b0815297dd03b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 18 Aug 2014 21:48:16 +0800
Subject: [PATCH 290/881] Thread tries to send data till stop() is called.

---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index c58ea363..a7730171 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -56,7 +56,7 @@ public void run() {
 					final Object[] data = new Object[dataLength];
 					int read = 1;
 					int written = 0;
-					while (read != 0 && !stopCalled) {
+					while (!stopCalled) {
 						read = readBuffer.read(data, 0, data.length);
 						written = 0;
 						while (written < read) {

From ba9ce1692c25467ec4dfc87cbf0ec896c85d8879 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 19 Aug 2014 08:41:22 +0800
Subject: [PATCH 291/881] busyWait -> sleep

Thread sleeps inside softClose() if writing msg fails.
---
 .../common/AsynchronousTCPConnection.java       | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
index a37e0c31..3f34dfea 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
@@ -165,8 +165,14 @@ public <T> T readObject() throws IOException, ClassNotFoundException {
 
 	@Override
 	public void softClose() throws IOException {
-		while (!bBAos.newWrite())
-			;
+		while (!bBAos.newWrite()) {
+			try {
+				// TODO : Find correct time for sleep.
+				Thread.sleep(100);
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
 		this.ooStream.write('\u001a');
 		this.ooStream.flush();
 		bBAos.writeCompleted();
@@ -590,6 +596,7 @@ public void write(byte b[], int off, int len) throws IOException {
 		public boolean newWrite() {
 			if (bufferStatus.get(writeIndex).compareAndSet(Status.canWrite,
 					Status.beingWritten)) {
+
 				if (debugLevel > 0)
 					System.out.println(Thread.currentThread().getName()
 							+ " : newWrite-canWrite : " + "writeIndex - "
@@ -618,9 +625,9 @@ public void writeCompleted() {
 			boolean ret = bufferStatus.get(w).compareAndSet(
 					Status.beingWritten, Status.canRead);
 			if (!ret) {
-				String msg = String.format("BufferState conflict : " + "writeIndex - "
-						+ writeIndex + ", readIndex - " + readIndex
-						+ " - Status of the writeBuffer is "
+				String msg = String.format("BufferState conflict : "
+						+ "writeIndex - " + writeIndex + ", readIndex - "
+						+ readIndex + " - Status of the writeBuffer is "
 						+ bufferStatus.get(w).get());
 				throw new IllegalStateException(msg);
 			}

From fb3d615441cc1ba46441975187ff7daa68af6776 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 19 Aug 2014 08:46:26 +0800
Subject: [PATCH 292/881] HeadThread calls stop().

Previously management thread (Opentuner thread ) was calling
AsynchronousTCPConnection.softClose() that caused synchronization problem
at writing data into Asynchronous socket buffer. Now same thread which writes
data to the socket calls the softClose as well.
---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index a7730171..0739bfdf 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -35,6 +35,7 @@ public static class AsyncTCPHeadChannel extends AsyncTCPOutputChannel {
 
 		final Buffer readBuffer;
 		private volatile boolean stopCalled;
+		private volatile boolean isFinal;
 
 		public AsyncTCPHeadChannel(Buffer buffer,
 				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
@@ -72,6 +73,7 @@ public void run() {
 							}
 						}
 					}
+					stopSuper(isFinal);
 				}
 			};
 		}
@@ -82,9 +84,13 @@ protected void fillUnprocessedData() {
 
 		@Override
 		public void stop(boolean isFinal) {
-			super.stop(isFinal);
+			this.isFinal = isFinal;
 			this.stopCalled = true;
 		}
+
+		private void stopSuper(boolean isFinal) {
+			super.stop(isFinal);
+		}
 	}
 
 	/**

From 65d0c458fcf9dd150367c728cfbbd43185475404 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 19 Aug 2014 11:25:01 +0800
Subject: [PATCH 293/881] Handles exception thrown by prepareDraining()

Catches the unchecked exceptions thrown by prepareDraining() and reset the
AbstractDrainer's state to NODRAINING.
---
 .../mit/streamjit/impl/common/AbstractDrainer.java | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 4f407c68..f11cc29e 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -160,27 +160,35 @@ public final void setBlobGraph(BlobGraph blobGraph) {
 	 */
 	public final boolean startDraining(int type) {
 		if (state == DrainerState.NODRAINING) {
+			boolean isFinal = false;
 			switch (type) {
 				case 0 :
 					this.blobGraph.clearDrainData();
 					this.state = DrainerState.INTERMEDIATE;
 					drainDataLatch = new CountDownLatch(1);
 					intermediateLatch = new CountDownLatch(1);
-					prepareDraining(false);
 					break;
 				case 1 :
 					this.state = DrainerState.FINAL;
-					prepareDraining(false);
 					break;
 				case 2 :
 					this.state = DrainerState.FINAL;
-					prepareDraining(true);
+					isFinal = true;
 					break;
 				default :
 					throw new IllegalArgumentException(
 							"Invalid draining type. type can be 0, 1, or 2.");
 			}
 
+			try {
+				prepareDraining(isFinal);
+			} catch (Exception e) {
+				this.state = DrainerState.NODRAINING;
+				System.err
+						.println("No Drain called. Exception in prepareDraining()");
+				throw e;
+			}
+
 			if (GlobalConstants.needDrainDeadlockHandler)
 				this.schExecutorService = Executors
 						.newSingleThreadScheduledExecutor();

From 128bad029037bf5fc671891cb98838eab5352458 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 19 Aug 2014 12:41:59 +0800
Subject: [PATCH 294/881] Closes safely if exp occurs.

Closes all StreamNodes and Controller safely if OpenTuner crashes.
---
 .../impl/distributed/runtimer/OnlineTuner.java | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 02a37e3d..b34adda3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -103,6 +103,7 @@ private void tune() {
 
 		} catch (IOException e) {
 			e.printStackTrace();
+			terminate();
 		}
 
 		try {
@@ -110,6 +111,9 @@ private void tune() {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
+
+		if (needTermination)
+			terminate();
 	}
 
 	private void evaluate() {
@@ -208,11 +212,7 @@ private void handleTermination() throws IOException {
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {
-			if (manager.isRunning()) {
-				drainer.startDraining(1);
-			} else {
-				manager.stop();
-			}
+			terminate();
 		} else {
 			Pair<Boolean, Long> ret = reconfigure(finalcfg);
 			if (ret.first && ret.second > 0)
@@ -223,6 +223,14 @@ private void handleTermination() throws IOException {
 		}
 	}
 
+	private void terminate() {
+		if (manager.isRunning()) {
+			drainer.startDraining(1);
+		} else {
+			manager.stop();
+		}
+	}
+
 	/**
 	 * Evaluates a configuration.
 	 * 

From f94e198eb8dd302d96b7ad12b7a3cf29cd32527c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 20 Aug 2014 17:20:16 +0800
Subject: [PATCH 295/881] Imports organized.

Used Eclipse's automatic import organizer to organize imports.
---
 .../impl/concurrent/ConcurrentDrainer.java         |  3 ++-
 .../impl/distributed/ConfigurationManager.java     |  6 +++---
 .../impl/distributed/ConnectionManager.java        |  2 +-
 .../impl/distributed/DistributedAppRunner.java     |  5 +----
 .../impl/distributed/DistributedBlobFactory.java   |  1 +
 .../distributed/DistributedStreamCompiler.java     |  2 +-
 .../streamjit/impl/distributed/HotSpotTuning.java  |  3 +--
 .../streamjit/impl/distributed/StreamJitApp.java   |  2 +-
 .../impl/distributed/StreamJitAppManager.java      | 12 ++++++------
 .../streamjit/impl/distributed/WorkerMachine.java  |  4 ++--
 .../distributed/common/BoundaryChannelManager.java |  3 ++-
 .../impl/distributed/common/TCPConnection.java     | 14 ++++++++++----
 .../streamjit/impl/distributed/common/Tester.java  |  4 ++--
 .../impl/distributed/node/BlobsManager.java        |  1 +
 .../impl/distributed/node/BlobsManagerImpl.java    | 13 ++++++++-----
 .../impl/distributed/node/BufferManager.java       |  2 +-
 .../distributed/node/CTRLRMessageVisitorImpl.java  |  8 ++++----
 .../distributed/node/CfgStringProcessorImpl.java   |  2 +-
 .../distributed/runtimer/CommunicationManager.java |  1 +
 .../impl/distributed/runtimer/Controller.java      |  3 +--
 .../test/apps/beamformer1/BeamFormer1.java         |  7 +++++++
 src/edu/mit/streamjit/tuner/OfflineTuner.java      |  1 -
 src/edu/mit/streamjit/tuner/RunApp2.java           | 11 ++++++-----
 23 files changed, 63 insertions(+), 47 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
index acd0e191..7b7c104a 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
@@ -1,9 +1,10 @@
 package edu.mit.streamjit.impl.concurrent;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.util.Map;
 import java.util.Set;
 
-import static com.google.common.base.Preconditions.*;
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.impl.blob.Blob;
diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index b4ab69c6..96333cbb 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -8,19 +8,19 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.Joiner;
 import edu.mit.streamjit.api.Splitter;
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index acf6f618..12311fe7 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -14,9 +14,9 @@
 import edu.mit.streamjit.impl.common.Configuration.Builder;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
 import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java b/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
index 5a42a848..870fb0ab 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedAppRunner.java
@@ -3,13 +3,10 @@
 import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.Output;
 import edu.mit.streamjit.api.StreamCompiler;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.interp.DebugStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
-import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.Benchmark.Dataset;
+import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
-import edu.mit.streamjit.test.apps.fmradio.FMRadio.FMRadioBenchmarkProvider;
 
 public class DistributedAppRunner {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 971b8ea5..1b79abdc 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -8,6 +8,7 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index b1757f33..a6afb7df 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -38,13 +38,13 @@
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
+import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.DistributedDrainer;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
-import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 29c24e4f..f77fbb8c 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -18,11 +18,10 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
+import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager.AbstractConfigurationManager;
-import edu.mit.streamjit.tuner.OfflineTuner;
 
 public final class HotSpotTuning extends AbstractConfigurationManager {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index ea3ebf60..fa92f629 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -11,10 +11,10 @@
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.MessageConstraint;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index a2b34ecd..3fa1c1c1 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -12,24 +12,24 @@
 
 import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
+import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Command;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
-import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error.ErrorProcessor;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.Drained;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 7dcc7e5d..22b2d1e2 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -11,12 +11,12 @@
 
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
+import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager.AbstractConfigurationManager;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
index e604222b..865c6d1e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
@@ -8,7 +8,8 @@
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.*;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 85e7619e..d082b05c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -1,12 +1,18 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import java.io.*;
-import java.net.*;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OptionalDataException;
+import java.net.InetAddress;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
-import static com.google.common.base.Preconditions.*;
-
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Tester.java b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
index 86b746ba..7a5ab51e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Tester.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
@@ -13,10 +13,10 @@
 import java.util.Map;
 
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 
 ;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
index 74848355..2a2987a4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f0e0f4d3..e5ac3642 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -18,28 +18,31 @@
 
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob;
-import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.*;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryInputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryOutputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.InputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.OutputChannelManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
-import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
-import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
+import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
-import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
+import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
  * {@link BlobsManagerImpl} responsible to run all {@link Blob}s those are
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index 5ad6d309..8d979b9b 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -10,9 +10,9 @@
 import com.google.common.collect.Sets;
 
 import edu.mit.streamjit.impl.blob.Blob;
+import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
-import edu.mit.streamjit.impl.blob.Blob.Token;
 
 /**
  * {@link BlobsManager} will use the services from {@link BufferManager}.
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index b4d55078..f9d49f94 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -3,17 +3,17 @@
 import java.io.IOException;
 
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageVisitor;
 import edu.mit.streamjit.impl.distributed.common.Command;
+import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements;
-import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.MiscCtrlElementProcessor;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
+import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
-import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
-import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.Request.RequestProcessor;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 44976174..8db6d5b6 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -31,9 +31,9 @@
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/CommunicationManager.java b/src/edu/mit/streamjit/impl/distributed/runtimer/CommunicationManager.java
index 9e6423e7..1756bd0d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/CommunicationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/CommunicationManager.java
@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.Map;
+
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index b494ed6a..9de2f11d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -9,16 +9,15 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
-import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
 
 /**
  * {@link Controller} controls all {@link StreamNode}s in runtime. It has
diff --git a/src/edu/mit/streamjit/test/apps/beamformer1/BeamFormer1.java b/src/edu/mit/streamjit/test/apps/beamformer1/BeamFormer1.java
index d0f92dff..041b0c6e 100644
--- a/src/edu/mit/streamjit/test/apps/beamformer1/BeamFormer1.java
+++ b/src/edu/mit/streamjit/test/apps/beamformer1/BeamFormer1.java
@@ -3,9 +3,12 @@
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.jeffreybosboom.serviceproviderprocessor.ServiceProvider;
+
+import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.DuplicateSplitter;
 import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.Input;
+import edu.mit.streamjit.api.Output;
 import edu.mit.streamjit.api.Pipeline;
 import edu.mit.streamjit.api.RoundrobinJoiner;
 import edu.mit.streamjit.api.RoundrobinSplitter;
@@ -13,12 +16,16 @@
 import edu.mit.streamjit.api.StatefulFilter;
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
+import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.interp.DebugStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmark.Dataset;
 import edu.mit.streamjit.test.Benchmarker;
+import edu.mit.streamjit.test.Datasets;
 import edu.mit.streamjit.test.SuppliedBenchmark;
 import java.nio.ByteOrder;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
 
diff --git a/src/edu/mit/streamjit/tuner/OfflineTuner.java b/src/edu/mit/streamjit/tuner/OfflineTuner.java
index c50b9358..f30fcb2d 100644
--- a/src/edu/mit/streamjit/tuner/OfflineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OfflineTuner.java
@@ -29,7 +29,6 @@
 import edu.mit.streamjit.test.Benchmark.Dataset;
 import edu.mit.streamjit.test.BenchmarkProvider;
 import edu.mit.streamjit.test.Datasets;
-import edu.mit.streamjit.test.apps.bitonicsort.BitonicSort;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
diff --git a/src/edu/mit/streamjit/tuner/RunApp2.java b/src/edu/mit/streamjit/tuner/RunApp2.java
index ca6cacee..3a59831a 100644
--- a/src/edu/mit/streamjit/tuner/RunApp2.java
+++ b/src/edu/mit/streamjit/tuner/RunApp2.java
@@ -1,16 +1,17 @@
 package edu.mit.streamjit.tuner;
 
-import edu.mit.streamjit.api.StreamCompiler;
-import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
-import edu.mit.streamjit.test.Benchmark;
-import edu.mit.streamjit.test.Benchmarker;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.List;
 
+import edu.mit.streamjit.api.StreamCompiler;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
+import edu.mit.streamjit.test.Benchmark;
+import edu.mit.streamjit.test.Benchmarker;
+
 /**
  * Runs a benchmark using a specified configuration.
  * @author Jeffrey Bosboom <jeffreybosboom@gmail.com>

From f5cbc5dc0a226b3c9f9b40cb8af2faf342b342aa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 15:53:10 +0800
Subject: [PATCH 296/881] Renaming:- getNewTCPConInfo() -> replaceConInfo()

---
 .../impl/distributed/ConnectionManager.java       | 15 +++++++++++++--
 .../impl/distributed/StreamJitAppManager.java     |  2 +-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 12311fe7..63748d02 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -21,6 +21,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
 import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
  * Generates configuration parameters to tune the {@link Connection}'s
@@ -66,7 +67,17 @@ public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap,
 			Worker<?, ?> source, Worker<?, ?> sink);
 
-	public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo);
+	/**
+	 * Sometimes an assigned TCP ports may not available to make new connection
+	 * at {@link StreamNode}s side. In this case a new {@link ConnectionInfo}
+	 * must be created to replace already created {@link ConnectionInfo}.
+	 * 
+	 * @param conInfo
+	 *            : Problematic {@link ConnectionInfo}.
+	 * @return : New {@link ConnectionInfo} to replace problematic
+	 *         {@link ConnectionInfo}.
+	 */
+	public ConnectionInfo replaceConInfo(ConnectionInfo conInfo);
 
 	public abstract static class AbstractConnectionManager implements
 			ConnectionManager {
@@ -168,7 +179,7 @@ protected String getParamName(Token t) {
 					t.getUpstreamIdentifier(), t.getDownstreamIdentifier());
 		}
 
-		public ConnectionInfo getNewTCPConInfo(ConnectionInfo conInfo) {
+		public ConnectionInfo replaceConInfo(ConnectionInfo conInfo) {
 			if (currentConInfos.contains(conInfo))
 				currentConInfos.remove(conInfo);
 			ConnectionInfo newConinfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 3fa1c1c1..cc8d6a16 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -406,7 +406,7 @@ public void process(AddressBindException abEx) {
 				}
 
 				ConnectionInfo coninfo = conManager
-						.getNewTCPConInfo(abEx.conInfo);
+						.replaceConInfo(abEx.conInfo);
 
 				exConInfos.add(abEx.conInfo);
 

From 0eabe815db373df50c5c0d8d390339d6b65f9227 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 16:20:22 +0800
Subject: [PATCH 297/881] addtoConInfoMap() gets cfg as an addtional arg

When we tune the connections, we need configuration to make new connections.
---
 .../impl/distributed/ConnectionManager.java       | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 63748d02..6074e77b 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -125,7 +125,7 @@ public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
 								partitionsMachineMap);
 						Token t = new Token(w, succ);
 						addtoconInfoMap(machineID, dstMachineID, t,
-								usedConInfos, conInfoMap);
+								usedConInfos, conInfoMap, cfg);
 					}
 				}
 			}
@@ -133,12 +133,12 @@ public Map<Token, ConnectionInfo> conInfoMap(Configuration cfg,
 			Token headToken = Token.createOverallInputToken(source);
 			int dstMachineID = getAssignedMachine(source, partitionsMachineMap);
 			addtoconInfoMap(controllerNodeID, dstMachineID, headToken,
-					usedConInfos, conInfoMap);
+					usedConInfos, conInfoMap, cfg);
 
 			Token tailToken = Token.createOverallOutputToken(sink);
 			int srcMahineID = getAssignedMachine(sink, partitionsMachineMap);
 			addtoconInfoMap(srcMahineID, controllerNodeID, tailToken,
-					usedConInfos, conInfoMap);
+					usedConInfos, conInfoMap, cfg);
 
 			return conInfoMap;
 		}
@@ -163,7 +163,7 @@ private int getAssignedMachine(Worker<?, ?> worker,
 
 		protected abstract void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,
-				Map<Token, ConnectionInfo> conInfoMap);
+				Map<Token, ConnectionInfo> conInfoMap, Configuration cfg);
 
 		protected List<ConnectionInfo> getTcpConInfo(ConnectionInfo conInfo) {
 			List<ConnectionInfo> conList = new ArrayList<>();
@@ -214,7 +214,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		protected void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,
-				Map<Token, ConnectionInfo> conInfoMap) {
+				Map<Token, ConnectionInfo> conInfoMap, Configuration cfg) {
 
 			ConnectionInfo conInfo = new GenericConnectionInfo(srcID, dstID);
 
@@ -270,7 +270,6 @@ protected ConnectionInfo makeConnectionInfo(int srcID, int dstID,
 	public static class AllConnectionParams extends AbstractConnectionManager {
 		public AllConnectionParams(int controllerNodeID) {
 			super(controllerNodeID);
-			// TODO Auto-generated constructor stub
 		}
 
 		@Override
@@ -310,9 +309,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		@Override
 		protected void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,
-				Map<Token, ConnectionInfo> conInfoMap) {
-			// TODO Auto-generated method stub
-
+				Map<Token, ConnectionInfo> conInfoMap, Configuration cfg) {
 		}
 	}
 }

From b5817be8832a4034f75dee98fa52f7cfa1a8533b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 16:32:40 +0800
Subject: [PATCH 298/881] The arg portNo removed from makeConnectionInfo().

Not all subclasses need portNo to make new connection Info.
---
 .../impl/distributed/ConnectionManager.java        | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 6074e77b..8417cdd0 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -229,7 +229,7 @@ protected void addtoconInfoMap(int srcID, int dstID, Token t,
 			}
 
 			if (tcpConInfo == null) {
-				tcpConInfo = makeConnectionInfo(srcID, dstID, startPortNo++);
+				tcpConInfo = makeConnectionInfo(srcID, dstID);
 				this.currentConInfos.add(tcpConInfo);
 			}
 
@@ -238,7 +238,7 @@ protected void addtoconInfoMap(int srcID, int dstID, Token t,
 		}
 
 		protected abstract ConnectionInfo makeConnectionInfo(int srcID,
-				int dstID, int portNo);
+				int dstID);
 	}
 
 	public static class BlockingTCPNoParams extends NoParams {
@@ -248,9 +248,8 @@ public BlockingTCPNoParams(int controllerNodeID) {
 		}
 
 		@Override
-		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID,
-				int portNo) {
-			return new TCPConnectionInfo(srcID, dstID, portNo);
+		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID) {
+			return new TCPConnectionInfo(srcID, dstID, startPortNo++);
 		}
 	}
 
@@ -261,9 +260,8 @@ public AsyncTCPNoParams(int controllerNodeID) {
 		}
 
 		@Override
-		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID,
-				int portNo) {
-			return new AsyncTCPConnectionInfo(srcID, dstID, portNo);
+		protected ConnectionInfo makeConnectionInfo(int srcID, int dstID) {
+			return new AsyncTCPConnectionInfo(srcID, dstID, startPortNo++);
 		}
 	}
 

From 43d18622affec0546929fea4c9ca018f335b8a86 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 18:33:39 +0800
Subject: [PATCH 299/881] Adds tuning parameter for global output channel.

---
 .../impl/distributed/ConnectionManager.java          | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 8417cdd0..651bff42 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -295,6 +295,18 @@ public void addChannelParameters(Builder cfgBuilder,
 								.values()));
 				cfgBuilder.addParameter(p);
 			}
+
+			// Add Parameter for global output channel.
+			Set<Worker<?, ?>> tail = Workers.getBottommostWorkers(workers);
+			assert tail.size() == 1 : "Multiple first workers";
+			for (Worker<?, ?> lastWorker : tail) {
+				Token t = Token.createOverallOutputToken(lastWorker);
+				Parameter p = new Configuration.SwitchParameter<ConnectionType>(
+						getParamName(t), ConnectionType.class,
+						ConnectionType.BTCP, Arrays.asList(ConnectionType
+								.values()));
+				cfgBuilder.addParameter(p);
+			}
 		}
 
 		@Override

From d869af1a65d513b86c653ea1463b7e9aef229e9c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 18:34:19 +0800
Subject: [PATCH 300/881] Creates connectionInfo based on OT's cfg.

---
 .../impl/distributed/ConnectionManager.java   | 48 ++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 651bff42..740b9d7c 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -13,6 +13,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.Builder;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
@@ -316,10 +317,55 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			return cfgBuilder.build();
 		}
 
-		@Override
 		protected void addtoconInfoMap(int srcID, int dstID, Token t,
 				Set<ConnectionInfo> usedConInfos,
 				Map<Token, ConnectionInfo> conInfoMap, Configuration cfg) {
+
+			ConnectionInfo conInfo = new GenericConnectionInfo(srcID, dstID);
+
+			List<ConnectionInfo> conSet = getTcpConInfo(conInfo);
+			ConnectionInfo tcpConInfo = null;
+
+			for (ConnectionInfo con : conSet) {
+				if (!usedConInfos.contains(con)) {
+					tcpConInfo = con;
+					break;
+				}
+			}
+
+			if (tcpConInfo == null) {
+				tcpConInfo = makeConnectionInfo(srcID, dstID, t, cfg);
+				this.currentConInfos.add(tcpConInfo);
+			}
+
+			conInfoMap.put(t, tcpConInfo);
+			usedConInfos.add(tcpConInfo);
+		}
+
+		private ConnectionInfo makeConnectionInfo(int srcID, int dstID,
+				Token t, Configuration cfg) {
+			SwitchParameter<ConnectionType> p = cfg.getParameter(
+					getParamName(t), SwitchParameter.class,
+					ConnectionType.class);
+
+			if (p == null)
+				throw new IllegalStateException(String.format(
+						"No tuning parameter for connection %s", t));
+
+			ConnectionInfo conInfo;
+			switch (p.getValue()) {
+				case BTCP :
+					conInfo = new TCPConnectionInfo(srcID, dstID, startPortNo++);
+					break;
+				case ATCP :
+					conInfo = new AsyncTCPConnectionInfo(srcID, dstID,
+							startPortNo++);
+					break;
+				default :
+					throw new IllegalStateException(String.format(
+							"Unsupported connection type - %s", p.getValue()));
+			}
+			return conInfo;
 		}
 	}
 }

From 7c6f5c50c1343a9f665bb6830e5e8d9dd0f9d43e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 18:43:59 +0800
Subject: [PATCH 301/881] Unimplemented connection types have been commented

Blocking InfiniBand, Non-Blocking InfiniBand and Non-Blocking TCP options have
been commented out from ConnecionType.
---
 .../impl/distributed/common/Connection.java        | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 55da7b8e..0e6c87f2 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -255,16 +255,22 @@ public enum ConnectionType {
 		 */
 		BTCP, /**
 		 * Non-Blocking TCP socket connection
+		 * 
+		 * NBTCP,
 		 */
-		NBTCP, /**
+		/**
 		 * Asynchronous TCP socket connection
 		 */
-		ATCP, /**
+		ATCP,
+		/**
 		 * Blocking InfiniBand
+		 * 
+		 * BIB,
 		 */
-		BIB, /**
+		/**
 		 * Non-Blocking InfiniBand
+		 * 
+		 * NBIB
 		 */
-		NBIB
 	}
 }

From 704c2d6b97c9e1202e2647504bec9fdb385f30ac Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 21 Aug 2014 18:48:15 +0800
Subject: [PATCH 302/881] Method renaming

addChannelParameters() -> addConnectionParameters().
---
 .../mit/streamjit/impl/distributed/ConnectionManager.java | 8 ++++----
 .../impl/distributed/DistributedBlobFactory.java          | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 740b9d7c..4fb172d3 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -42,7 +42,7 @@ public interface ConnectionManager {
 	 * @param cfgBuilder
 	 * @param workers
 	 */
-	public void addChannelParameters(Configuration.Builder cfgBuilder,
+	public void addConnectionParameters(Configuration.Builder cfgBuilder,
 			Set<Worker<?, ?>> workers);
 
 	/**
@@ -203,7 +203,7 @@ public NoParams(int controllerNodeID) {
 		}
 
 		@Override
-		public void addChannelParameters(Builder cfgBuilder,
+		public void addConnectionParameters(Builder cfgBuilder,
 				Set<Worker<?, ?>> workers) {
 			return;
 		}
@@ -272,7 +272,7 @@ public AllConnectionParams(int controllerNodeID) {
 		}
 
 		@Override
-		public void addChannelParameters(Builder cfgBuilder,
+		public void addConnectionParameters(Builder cfgBuilder,
 				Set<Worker<?, ?>> workers) {
 			for (Worker<?, ?> w : workers) {
 				for (Worker<?, ?> succ : Workers.getSuccessors(w)) {
@@ -313,7 +313,7 @@ public void addChannelParameters(Builder cfgBuilder,
 		@Override
 		public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			Configuration.Builder cfgBuilder = Configuration.builder();
-			addChannelParameters(cfgBuilder, workers);
+			addConnectionParameters(cfgBuilder, workers);
 			return cfgBuilder.build();
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 1b79abdc..86618bff 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -77,7 +77,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		for (Parameter p : compilercfg.getParametersMap().values())
 			builder.addParameter(p);
 
-		connectionManager.addChannelParameters(builder, workers);
+		connectionManager.addConnectionParameters(builder, workers);
 		return builder.build();
 	}
 

From 7a421db1aae78d9fd5e5a24f46bee93954e4691a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 22 Aug 2014 21:53:06 +0800
Subject: [PATCH 303/881] Prints termination message

If OpenTuner crashes, Controller prints message before terminate.
---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b34adda3..79895d24 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -75,7 +75,10 @@ private void tune() {
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
 				if (cfgJson == null)
+				{
+					System.err.println("OpenTuner closed unexpectly.");
 					break;
+				}
 
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.

From a410ba85fcb98beafb816cc895c88f1996b8aadc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 5 Sep 2014 23:59:04 +0800
Subject: [PATCH 304/881] Obsolete file deleted.

ChannelManagers.java has been deleted. BoundaryChannelManager.InputChannelManager
and OutputChannelManager have updated implementation for BoundaryChannelManagers.
---
 .../distributed/node/ChannelManagers.java     | 137 ------------------
 1 file changed, 137 deletions(-)
 delete mode 100644 src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java b/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java
deleted file mode 100644
index 141f97ce..00000000
--- a/src/edu/mit/streamjit/impl/distributed/node/ChannelManagers.java
+++ /dev/null
@@ -1,137 +0,0 @@
-package edu.mit.streamjit.impl.distributed.node;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.google.common.collect.ImmutableMap;
-
-import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryInputChannelManager;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryOutputChannelManager;
-
-public class ChannelManagers {
-
-	public static class BlockingInputChannelManager
-			implements
-				BoundaryInputChannelManager {
-
-		private final ImmutableMap<Token, BoundaryInputChannel> inputChannels;
-
-		private final Set<Thread> inputChannelThreads;
-
-		public BlockingInputChannelManager(
-				final ImmutableMap<Token, BoundaryInputChannel> inputChannels) {
-			this.inputChannels = inputChannels;
-			inputChannelThreads = new HashSet<>(inputChannels.values().size());
-		}
-
-		@Override
-		public void start() {
-			for (BoundaryInputChannel bc : inputChannels.values()) {
-				Thread t = new Thread(bc.getRunnable(), bc.name());
-				t.start();
-				inputChannelThreads.add(t);
-			}
-		}
-
-		@Override
-		public void waitToStart() {
-		}
-
-		@Override
-		public void waitToStop() {
-			for (Thread t : inputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-
-		@Override
-		public void stop(int stopType) {
-			for (BoundaryInputChannel bc : inputChannels.values()) {
-				bc.stop(stopType);
-			}
-		}
-
-		@Override
-		public ImmutableMap<Token, BoundaryInputChannel> inputChannelsMap() {
-			return inputChannels;
-		}
-	}
-
-	public static class BlockingOutputChannelManager
-			implements
-				BoundaryOutputChannelManager {
-
-		protected final ImmutableMap<Token, BoundaryOutputChannel> outputChannels;
-		protected final Set<Thread> outputChannelThreads;
-
-		public BlockingOutputChannelManager(
-				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
-			this.outputChannels = outputChannels;
-			outputChannelThreads = new HashSet<>(outputChannels.values().size());
-		}
-
-		@Override
-		public void start() {
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
-				Thread t = new Thread(bc.getRunnable(), bc.name());
-				t.start();
-				outputChannelThreads.add(t);
-			}
-		}
-
-		@Override
-		public void waitToStart() {
-		}
-
-		@Override
-		public void stop(boolean stopType) {
-			for (BoundaryOutputChannel bc : outputChannels.values()) {
-				bc.stop(stopType);
-			}
-		}
-
-		@Override
-		public void waitToStop() {
-			for (Thread t : outputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-
-		@Override
-		public ImmutableMap<Token, BoundaryOutputChannel> outputChannelsMap() {
-			return outputChannels;
-		}
-	}
-
-	public static class AsynchronousOutputChannelManager
-			extends
-				BlockingOutputChannelManager {
-
-		public AsynchronousOutputChannelManager(
-				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
-			super(outputChannels);
-		}
-
-		@Override
-		public void waitToStart() {
-			for (Thread t : outputChannelThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-	}
-}

From fe0c0306a7b11801a605a02c19beea7a4d4385b0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 9 Sep 2014 06:58:01 +0800
Subject: [PATCH 305/881] Rename:AsynchronousTCPConnection->AsyncTCPConnection

---
 .../streamjit/impl/distributed/ConnectionManager.java  |  2 +-
 .../impl/distributed/StreamJitAppManager.java          |  2 +-
 ...onousTCPConnection.java => AsyncTCPConnection.java} | 10 +++++-----
 .../impl/distributed/common/ConnectionFactory.java     |  4 ++--
 .../mit/streamjit/impl/distributed/common/Tester.java  |  2 +-
 .../impl/distributed/node/AsyncTCPOutputChannel.java   |  6 +++---
 6 files changed, 13 insertions(+), 13 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/common/{AsynchronousTCPConnection.java => AsyncTCPConnection.java} (98%)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
index 4fb172d3..be47742d 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConnectionManager.java
@@ -15,7 +15,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index cc8d6a16..55ebd84a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -19,7 +19,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
similarity index 98%
rename from src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
rename to src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index 3f34dfea..6da00caa 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsynchronousTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -37,7 +37,7 @@
  * @since May 05, 2014
  * 
  */
-public class AsynchronousTCPConnection implements Connection {
+public class AsyncTCPConnection implements Connection {
 	/**
 	 * Backed by {@link ByteBufferArrayOutputStream}.
 	 */
@@ -49,7 +49,7 @@ public class AsynchronousTCPConnection implements Connection {
 
 	private boolean isconnected = false;
 
-	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
+	public AsyncTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
 		this(asyncSktChannel, 5000);
 	}
 
@@ -59,7 +59,7 @@ public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel) {
 	 *            reset the {@link ObjectOutputStream} after this no of sends.
 	 *            To avoid out of memory error.
 	 */
-	public AsynchronousTCPConnection(AsynchronousSocketChannel asyncSktChannel,
+	public AsyncTCPConnection(AsynchronousSocketChannel asyncSktChannel,
 			int resetCount) {
 		try {
 			this.asyncSktChannel = asyncSktChannel;
@@ -783,9 +783,9 @@ public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 
 	public static class AsyncTCPBuffer extends AbstractWriteOnlyBuffer {
 
-		private final AsynchronousTCPConnection con;
+		private final AsyncTCPConnection con;
 
-		public AsyncTCPBuffer(AsynchronousTCPConnection con) {
+		public AsyncTCPBuffer(AsyncTCPConnection con) {
 			this.con = con;
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
index ac9b3db5..99fc671e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConnectionFactory.java
@@ -85,7 +85,7 @@ public static Connection getConnection(Socket socket, boolean needSync)
 			return new TCPConnection(socket);
 	}
 
-	public static AsynchronousTCPConnection getAsyncConnection(int portNo)
+	public static AsyncTCPConnection getAsyncConnection(int portNo)
 			throws IOException {
 		AsynchronousServerSocketChannel ssc;
 		AsynchronousSocketChannel sc2;
@@ -103,6 +103,6 @@ public static AsynchronousTCPConnection getAsyncConnection(int portNo)
 		}
 
 		ssc.close();
-		return new AsynchronousTCPConnection(sc2);
+		return new AsyncTCPConnection(sc2);
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Tester.java b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
index 7a5ab51e..aa2746fe 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Tester.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Tester.java
@@ -12,7 +12,7 @@
 import java.util.List;
 import java.util.Map;
 
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionType;
 import edu.mit.streamjit.impl.distributed.common.Connection.GenericConnectionInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
index 3cf522ea..90452c49 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
@@ -5,8 +5,8 @@
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection;
-import edu.mit.streamjit.impl.distributed.common.AsynchronousTCPConnection.AsyncTCPBuffer;
+import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection;
+import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPBuffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
@@ -51,7 +51,7 @@ public void run() {
 					try {
 						con = conProvider.getConnection(conInfo);
 						buffer = new AsyncTCPBuffer(
-								(AsynchronousTCPConnection) con);
+								(AsyncTCPConnection) con);
 					} catch (IOException e) {
 						e.printStackTrace();
 					}

From 2d42c08628282b083455401247407fd7572ee0dd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 9 Sep 2014 07:03:49 +0800
Subject: [PATCH 306/881] Renamed a variable : tcpConnection -> connection

---
 .../impl/distributed/node/TCPInputChannel.java | 16 ++++++++--------
 .../distributed/node/TCPOutputChannel.java     | 18 +++++++++---------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
index 0438e4f7..28d0b3c4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
@@ -47,7 +47,7 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private final ConnectionInfo conInfo;
 
-	private Connection tcpConnection;
+	private Connection connection;
 
 	private final AtomicInteger stopType;
 
@@ -104,9 +104,9 @@ public Runnable getRunnable() {
 		return new Runnable() {
 			@Override
 			public void run() {
-				if (tcpConnection == null || !tcpConnection.isStillConnected()) {
+				if (connection == null || !connection.isStillConnected()) {
 					try {
-						tcpConnection = conProvider.getConnection(conInfo);
+						connection = conProvider.getConnection(conInfo);
 					} catch (IOException e) {
 						// TODO: Need to handle this exception.
 						e.printStackTrace();
@@ -145,7 +145,7 @@ public void run() {
 	public void receiveData() {
 		int bufFullCount = 0;
 		try {
-			Object obj = tcpConnection.readObject();
+			Object obj = connection.readObject();
 			if (obj == null) // [2014-03-15] Sometimes null is received.
 				return;
 			count++;
@@ -230,7 +230,7 @@ private void finalReceive() {
 		do {
 			bufFullCount = 0;
 			try {
-				Object obj = tcpConnection.readObject();
+				Object obj = connection.readObject();
 				count++;
 
 				if (debugLevel == 2) {
@@ -308,7 +308,7 @@ private void discardAll() {
 		boolean hasData;
 		do {
 			try {
-				Object obj = tcpConnection.readObject();
+				Object obj = connection.readObject();
 				hasData = true;
 			} catch (ClassNotFoundException e) {
 				hasData = true;
@@ -328,8 +328,8 @@ private void reConnect() {
 		while (stopType.get() == 0) {
 			try {
 				System.out.println("TCPInputChannel : Reconnecting...");
-				this.tcpConnection.closeConnection();
-				tcpConnection = conProvider.getConnection(conInfo);
+				this.connection.closeConnection();
+				connection = conProvider.getConnection(conInfo);
 				return;
 			} catch (IOException e) {
 				try {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
index 1d0b38d5..0edaa4b2 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
@@ -38,7 +38,7 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	private final ConnectionInfo conInfo;
 
-	private Connection tcpConnection;
+	private Connection connection;
 
 	private final AtomicBoolean stopFlag;
 
@@ -83,7 +83,7 @@ public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 
 	private void closeConnection() throws IOException {
 		// tcpConnection.closeConnection();
-		tcpConnection.softClose();
+		connection.softClose();
 	}
 
 	@Override
@@ -91,9 +91,9 @@ public final Runnable getRunnable() {
 		return new Runnable() {
 			@Override
 			public void run() {
-				if (tcpConnection == null || !tcpConnection.isStillConnected()) {
+				if (connection == null || !connection.isStillConnected()) {
 					try {
-						tcpConnection = conProvider.getConnection(conInfo);
+						connection = conProvider.getConnection(conInfo);
 					} catch (IOException e) {
 						e.printStackTrace();
 					}
@@ -134,7 +134,7 @@ public final void sendData() {
 		while (this.buffer.size() > 0 && !stopFlag.get()) {
 			try {
 				Object obj = buffer.read();
-				tcpConnection.writeObject(obj);
+				connection.writeObject(obj);
 				count++;
 
 				if (debugLevel == 3) {
@@ -179,7 +179,7 @@ private void finalSend() {
 		while (this.buffer.size() > 0) {
 			try {
 				Object o = buffer.read();
-				tcpConnection.writeObject(o);
+				connection.writeObject(o);
 				count++;
 
 				if (debugLevel == 3) {
@@ -205,11 +205,11 @@ private void finalSend() {
 
 	private void reConnect() {
 		try {
-			this.tcpConnection.closeConnection();
+			this.connection.closeConnection();
 			while (!stopFlag.get()) {
 				System.out.println("TCPOutputChannel : Reconnecting...");
 				try {
-					this.tcpConnection = conProvider.getConnection(conInfo,
+					this.connection = conProvider.getConnection(conInfo,
 							1000);
 					return;
 				} catch (SocketTimeoutException stex) {
@@ -247,7 +247,7 @@ public ImmutableList<Object> getUnprocessedData() {
 
 	@Override
 	public Connection getConnection() {
-		return tcpConnection;
+		return connection;
 	}
 
 	@Override

From 27e0a98f5c1bfb4e356b5e879e6e192e9523fbbf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 9 Sep 2014 07:50:06 +0800
Subject: [PATCH 307/881] Rename : TCPInputChannel -> BlockingInputChannel

---
 .../mit/streamjit/impl/distributed/TailChannel.java    |  4 ++--
 .../impl/distributed/common/AsyncTCPConnection.java    |  4 ++--
 .../distributed/common/BoundaryChannelFactory.java     | 10 +++++-----
 .../impl/distributed/common/TCPConnection.java         |  4 ++--
 ...{TCPInputChannel.java => BlockingInputChannel.java} |  6 +++---
 5 files changed, 14 insertions(+), 14 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/node/{TCPInputChannel.java => BlockingInputChannel.java} (98%)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 0377ba87..1c618eeb 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -13,9 +13,9 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
-public class TailChannel extends TCPInputChannel {
+public class TailChannel extends BlockingInputChannel {
 
 	private final int skipCount;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index 6da00caa..d478f8aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -20,7 +20,7 @@
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
 /**
  * Uses {@link AsynchronousSocketChannel} from Java's NIO.2 to send data. This
@@ -770,7 +770,7 @@ else if (dstID == nodeID) {
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			return new TCPInputChannel(bufSize, conProvider, this,
+			return new BlockingInputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index 231d4306..20427e92 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -7,7 +7,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
-import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
 
 /**
@@ -31,7 +31,7 @@ BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 			ConnectionInfo conInfo);
 
 	/**
-	 * Makes blocking {@link TCPInputChannel} and {@link TCPOutputChannel}.
+	 * Makes blocking {@link BlockingInputChannel} and {@link TCPOutputChannel}.
 	 * 
 	 */
 	public static class TCPBoundaryChannelFactory
@@ -47,7 +47,7 @@ public TCPBoundaryChannelFactory(TCPConnectionProvider conProvider) {
 		@Override
 		public BoundaryInputChannel makeInputChannel(Token t, Buffer buffer,
 				ConnectionInfo conInfo) {
-			return new TCPInputChannel(buffer, conProvider, conInfo,
+			return new BlockingInputChannel(buffer, conProvider, conInfo,
 					t.toString(), 0);
 		}
 
@@ -61,7 +61,7 @@ public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 		@Override
 		public BoundaryInputChannel makeInputChannel(Token t, int bufSize,
 				ConnectionInfo conInfo) {
-			return new TCPInputChannel(bufSize, conProvider, conInfo,
+			return new BlockingInputChannel(bufSize, conProvider, conInfo,
 					t.toString(), 0);
 		}
 
@@ -74,7 +74,7 @@ public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 	}
 
 	/**
-	 * Makes blocking {@link TCPInputChannel} and asynchronous
+	 * Makes blocking {@link BlockingInputChannel} and asynchronous
 	 * {@link AsyncTCPOutputChannel}.
 	 * 
 	 */
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index d082b05c..40a2cf5a 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.node.TCPInputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
 
 /**
@@ -258,7 +258,7 @@ else if (dstID == nodeID) {
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			return new TCPInputChannel(bufSize, conProvider, this,
+			return new BlockingInputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
similarity index 98%
rename from src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
rename to src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 28d0b3c4..d478058b 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -33,7 +33,7 @@
  * @author Sumanan sumanan@mit.edu
  * @since May 29, 2013
  */
-public class TCPInputChannel implements BoundaryInputChannel {
+public class BlockingInputChannel implements BoundaryInputChannel {
 
 	private final FileWriter writer;
 
@@ -61,13 +61,13 @@ public class TCPInputChannel implements BoundaryInputChannel {
 
 	private ImmutableList<Object> unProcessedData;
 
-	public TCPInputChannel(int bufSize, TCPConnectionProvider conProvider,
+	public BlockingInputChannel(int bufSize, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
 				bufferTokenName, debugLevel);
 	}
 
-	public TCPInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
+	public BlockingInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;

From 0668a7b5d81846e62587bc0249de78e7993ba74f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 9 Sep 2014 07:59:03 +0800
Subject: [PATCH 308/881] Rename : TCPOutputChannel -> BlockingOutputChannel

---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java   | 4 ++--
 .../impl/distributed/common/BoundaryChannelFactory.java   | 8 ++++----
 .../streamjit/impl/distributed/common/TCPConnection.java  | 4 ++--
 .../{TCPOutputChannel.java => BlockingOutputChannel.java} | 6 +++---
 4 files changed, 11 insertions(+), 11 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/node/{TCPOutputChannel.java => BlockingOutputChannel.java} (96%)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 0739bfdf..803135aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
-import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
 /**
  * Head Channel is just a wrapper to TCPOutputChannel that skips
@@ -19,7 +19,7 @@
  */
 public class HeadChannel {
 
-	public static class TCPHeadChannel extends TCPOutputChannel {
+	public static class TCPHeadChannel extends BlockingOutputChannel {
 
 		public TCPHeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
 				ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index 20427e92..474776f8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
-import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
 /**
  * {@link BoundaryChannel} maker.
@@ -31,7 +31,7 @@ BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 			ConnectionInfo conInfo);
 
 	/**
-	 * Makes blocking {@link BlockingInputChannel} and {@link TCPOutputChannel}.
+	 * Makes blocking {@link BlockingInputChannel} and {@link BlockingOutputChannel}.
 	 * 
 	 */
 	public static class TCPBoundaryChannelFactory
@@ -54,7 +54,7 @@ public BoundaryInputChannel makeInputChannel(Token t, Buffer buffer,
 		@Override
 		public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 				ConnectionInfo conInfo) {
-			return new TCPOutputChannel(buffer, conProvider, conInfo,
+			return new BlockingOutputChannel(buffer, conProvider, conInfo,
 					t.toString(), 0);
 		}
 
@@ -68,7 +68,7 @@ public BoundaryInputChannel makeInputChannel(Token t, int bufSize,
 		@Override
 		public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 				ConnectionInfo conInfo) {
-			return new TCPOutputChannel(bufSize, conProvider, conInfo,
+			return new BlockingOutputChannel(bufSize, conProvider, conInfo,
 					t.toString(), 0);
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 40a2cf5a..a5b7f996 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -18,7 +18,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
-import edu.mit.streamjit.impl.distributed.node.TCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
 /**
  * TCPConnection is not thread safe.
@@ -265,7 +265,7 @@ public BoundaryInputChannel inputChannel(Token t, int bufSize,
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			return new TCPOutputChannel(bufSize, conProvider, this,
+			return new BlockingOutputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
similarity index 96%
rename from src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
rename to src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index 0edaa4b2..87136867 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/TCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -26,7 +26,7 @@
  * @author Sumanan sumanan@mit.edu
  * @since May 29, 2013
  */
-public class TCPOutputChannel implements BoundaryOutputChannel {
+public class BlockingOutputChannel implements BoundaryOutputChannel {
 
 	private final FileWriter writer;
 
@@ -50,13 +50,13 @@ public class TCPOutputChannel implements BoundaryOutputChannel {
 
 	protected ImmutableList<Object> unProcessedData;
 
-	public TCPOutputChannel(int bufSize, TCPConnectionProvider conProvider,
+	public BlockingOutputChannel(int bufSize, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
 				bufferTokenName, debugLevel);
 	}
 
-	public TCPOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
+	public BlockingOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;

From fee4f642b9e491f912a62389aa6f920f1bebcc00 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 9 Sep 2014 09:14:19 +0800
Subject: [PATCH 309/881] Rename:AsyncTCPOutputChannel->AsyncOutputChannel

---
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java   | 6 +++---
 .../streamjit/impl/distributed/StreamJitAppManager.java   | 2 +-
 .../impl/distributed/common/AsyncTCPConnection.java       | 4 ++--
 .../impl/distributed/common/BoundaryChannelFactory.java   | 8 ++++----
 .../impl/distributed/common/BoundaryChannelManager.java   | 4 ++--
 ...AsyncTCPOutputChannel.java => AsyncOutputChannel.java} | 4 ++--
 6 files changed, 14 insertions(+), 14 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/node/{AsyncTCPOutputChannel.java => AsyncOutputChannel.java} (94%)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index 803135aa..d39d8fa4 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -7,7 +7,7 @@
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
 /**
@@ -31,13 +31,13 @@ protected void fillUnprocessedData() {
 		}
 	}
 
-	public static class AsyncTCPHeadChannel extends AsyncTCPOutputChannel {
+	public static class AsyncHeadChannel extends AsyncOutputChannel {
 
 		final Buffer readBuffer;
 		private volatile boolean stopCalled;
 		private volatile boolean isFinal;
 
-		public AsyncTCPHeadChannel(Buffer buffer,
+		public AsyncHeadChannel(Buffer buffer,
 				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel) {
 			super(conProvider, conInfo, bufferTokenName, debugLevel);
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 55ebd84a..b1b6cbbd 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -180,7 +180,7 @@ private void setupHeadTail(Map<Token, ConnectionInfo> conInfoMap,
 					bufferMap.get(headToken), controller.getConProvider(),
 					headconInfo, "headChannel - " + headToken.toString(), 0);
 		else if (headconInfo instanceof AsyncTCPConnectionInfo)
-			headChannel = new HeadChannel.AsyncTCPHeadChannel(
+			headChannel = new HeadChannel.AsyncHeadChannel(
 					bufferMap.get(headToken), controller.getConProvider(),
 					headconInfo, "headChannel - " + headToken.toString(), 0);
 		else
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index d478f8aa..d2bad51d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -18,7 +18,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
@@ -777,7 +777,7 @@ public BoundaryInputChannel inputChannel(Token t, int bufSize,
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 				TCPConnectionProvider conProvider) {
-			return new AsyncTCPOutputChannel(conProvider, this, t.toString(), 0);
+			return new AsyncOutputChannel(conProvider, this, t.toString(), 0);
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index 474776f8..f71e9b75 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -6,7 +6,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
-import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
@@ -75,7 +75,7 @@ public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 
 	/**
 	 * Makes blocking {@link BlockingInputChannel} and asynchronous
-	 * {@link AsyncTCPOutputChannel}.
+	 * {@link AsyncOutputChannel}.
 	 * 
 	 */
 	public class AsyncBoundaryChannelFactory extends TCPBoundaryChannelFactory {
@@ -87,14 +87,14 @@ public AsyncBoundaryChannelFactory(TCPConnectionProvider conProvider) {
 		@Override
 		public BoundaryOutputChannel makeOutputChannel(Token t, Buffer buffer,
 				ConnectionInfo conInfo) {
-			return new AsyncTCPOutputChannel(conProvider, conInfo,
+			return new AsyncOutputChannel(conProvider, conInfo,
 					t.toString(), 0);
 		}
 
 		@Override
 		public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 				ConnectionInfo conInfo) {
-			return new AsyncTCPOutputChannel(conProvider, conInfo,
+			return new AsyncOutputChannel(conProvider, conInfo,
 					t.toString(), 0);
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
index 865c6d1e..8e0a2993 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
@@ -10,7 +10,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.node.AsyncTCPOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 
 /**
  * Manages set of {@link BoundaryChannel}s.
@@ -139,7 +139,7 @@ public void start() {
 		public void waitToStart() {
 			for (Map.Entry<BoundaryOutputChannel, Thread> en : outputChannelThreads
 					.entrySet()) {
-				if (en.getKey() instanceof AsyncTCPOutputChannel) {
+				if (en.getKey() instanceof AsyncOutputChannel) {
 					try {
 						en.getValue().join();
 					} catch (InterruptedException e) {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
similarity index 94%
rename from src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
rename to src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
index 90452c49..2ecf7ff6 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncTCPOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
@@ -12,7 +12,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
 
-public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
+public class AsyncOutputChannel implements BoundaryOutputChannel {
 
 	private volatile Connection con;
 
@@ -28,7 +28,7 @@ public class AsyncTCPOutputChannel implements BoundaryOutputChannel {
 
 	private volatile boolean stopCalled;
 
-	public AsyncTCPOutputChannel(TCPConnectionProvider conProvider,
+	public AsyncOutputChannel(TCPConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		name = "AsyncTCPOutputChannel " + bufferTokenName;
 		this.conProvider = conProvider;

From 0f02b29291a9a602b7909a377e63179b0595db6e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 06:54:33 +0800
Subject: [PATCH 310/881] Renamed:TCPConnectionProvider->ConnectionProvider

The prefix TCP has no meaning after the latest changes in the class. This class
handles all kind of connections.
---
 .../mit/streamjit/impl/distributed/HeadChannel.java    |  6 +++---
 .../mit/streamjit/impl/distributed/TailChannel.java    |  4 ++--
 .../impl/distributed/common/AsyncTCPConnection.java    |  6 +++---
 .../distributed/common/BoundaryChannelFactory.java     |  8 ++++----
 .../streamjit/impl/distributed/common/Connection.java  | 10 +++++-----
 .../impl/distributed/common/TCPConnection.java         |  8 ++++----
 .../impl/distributed/node/AsyncOutputChannel.java      |  6 +++---
 .../impl/distributed/node/BlobsManagerImpl.java        |  6 +++---
 .../impl/distributed/node/BlockingInputChannel.java    |  8 ++++----
 .../impl/distributed/node/BlockingOutputChannel.java   |  8 ++++----
 .../impl/distributed/node/CfgStringProcessorImpl.java  |  6 +++---
 .../impl/distributed/runtimer/Controller.java          |  8 ++++----
 12 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index d39d8fa4..ee7a5a06 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -6,7 +6,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
@@ -21,7 +21,7 @@ public class HeadChannel {
 
 	public static class TCPHeadChannel extends BlockingOutputChannel {
 
-		public TCPHeadChannel(Buffer buffer, TCPConnectionProvider conProvider,
+		public TCPHeadChannel(Buffer buffer, ConnectionProvider conProvider,
 				ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 		}
@@ -38,7 +38,7 @@ public static class AsyncHeadChannel extends AsyncOutputChannel {
 		private volatile boolean isFinal;
 
 		public AsyncHeadChannel(Buffer buffer,
-				TCPConnectionProvider conProvider, ConnectionInfo conInfo,
+				ConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel) {
 			super(conProvider, conInfo, bufferTokenName, debugLevel);
 			readBuffer = buffer;
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 1c618eeb..38a23aad 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -12,7 +12,7 @@
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
 public class TailChannel extends BlockingInputChannel {
@@ -47,7 +47,7 @@ public class TailChannel extends BlockingInputChannel {
 	 *            get this amount of outputs ( after skipping skipCount number
 	 *            of outputs at the beginning).
 	 */
-	public TailChannel(Buffer buffer, TCPConnectionProvider conProvider,
+	public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int skipCount, int steadyCount) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index d2bad51d..91f27cd6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
@@ -769,14 +769,14 @@ else if (dstID == nodeID) {
 
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			return new BlockingInputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
 
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			return new AsyncOutputChannel(conProvider, this, t.toString(), 0);
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index f71e9b75..0da08c9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -5,7 +5,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
@@ -38,9 +38,9 @@ public static class TCPBoundaryChannelFactory
 			implements
 				BoundaryChannelFactory {
 
-		protected final TCPConnectionProvider conProvider;
+		protected final ConnectionProvider conProvider;
 
-		public TCPBoundaryChannelFactory(TCPConnectionProvider conProvider) {
+		public TCPBoundaryChannelFactory(ConnectionProvider conProvider) {
 			this.conProvider = conProvider;
 		}
 
@@ -80,7 +80,7 @@ public BoundaryOutputChannel makeOutputChannel(Token t, int bufSize,
 	 */
 	public class AsyncBoundaryChannelFactory extends TCPBoundaryChannelFactory {
 
-		public AsyncBoundaryChannelFactory(TCPConnectionProvider conProvider) {
+		public AsyncBoundaryChannelFactory(ConnectionProvider conProvider) {
 			super(conProvider);
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index 0e6c87f2..ad670e14 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -7,7 +7,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -199,10 +199,10 @@ public abstract Connection makeConnection(int nodeID,
 				NetworkInfo networkInfo, int timeOut);
 
 		public abstract BoundaryInputChannel inputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider);
+				ConnectionProvider conProvider);
 
 		public abstract BoundaryOutputChannel outputChannel(Token t,
-				int bufSize, TCPConnectionProvider conProvider);
+				int bufSize, ConnectionProvider conProvider);
 	}
 
 	/**
@@ -230,13 +230,13 @@ public Connection makeConnection(int nodeID, NetworkInfo networkInfo,
 
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			throw new java.lang.Error("This method is not supposed to call");
 		}
 
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			throw new java.lang.Error("This method is not supposed to call");
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index a5b7f996..07b31a83 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -257,14 +257,14 @@ else if (dstID == nodeID) {
 
 		@Override
 		public BoundaryInputChannel inputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			return new BlockingInputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
 
 		@Override
 		public BoundaryOutputChannel outputChannel(Token t, int bufSize,
-				TCPConnectionProvider conProvider) {
+				ConnectionProvider conProvider) {
 			return new BlockingOutputChannel(bufSize, conProvider, this,
 					t.toString(), 0);
 		}
@@ -281,7 +281,7 @@ public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 	 * same JVM. So first, local {@link StreamNode} should be made to run on a
 	 * different JVM and then make this class singleton.
 	 */
-	public static class TCPConnectionProvider {
+	public static class ConnectionProvider {
 
 		private ConcurrentMap<ConnectionInfo, Connection> allConnections;
 
@@ -289,7 +289,7 @@ public static class TCPConnectionProvider {
 
 		private final NetworkInfo networkInfo;
 
-		public TCPConnectionProvider(int myNodeID, NetworkInfo networkInfo) {
+		public ConnectionProvider(int myNodeID, NetworkInfo networkInfo) {
 			checkNotNull(networkInfo, "networkInfo is null");
 			this.myNodeID = myNodeID;
 			this.networkInfo = networkInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
index 2ecf7ff6..3f5f3328 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
@@ -10,7 +10,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 
 public class AsyncOutputChannel implements BoundaryOutputChannel {
 
@@ -18,7 +18,7 @@ public class AsyncOutputChannel implements BoundaryOutputChannel {
 
 	private final String name;
 
-	private final TCPConnectionProvider conProvider;
+	private final ConnectionProvider conProvider;
 
 	private final ConnectionInfo conInfo;
 
@@ -28,7 +28,7 @@ public class AsyncOutputChannel implements BoundaryOutputChannel {
 
 	private volatile boolean stopCalled;
 
-	public AsyncOutputChannel(TCPConnectionProvider conProvider,
+	public AsyncOutputChannel(ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		name = "AsyncTCPOutputChannel " + bufferTokenName;
 		this.conProvider = conProvider;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e5ac3642..d9772120 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -39,7 +39,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -67,7 +67,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final BufferManager bufferManager;
 
-	private final TCPConnectionProvider conProvider;
+	private final ConnectionProvider conProvider;
 
 	/**
 	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
@@ -78,7 +78,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
-			TCPConnectionProvider conProvider) {
+			ConnectionProvider conProvider) {
 		this.conInfoMap = conInfoMap;
 		this.streamNode = streamNode;
 		this.conProvider = conProvider;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index d478058b..39c25b8e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 
 /**
  * This is {@link BoundaryInputChannel} over TCP. Receive objects from TCP
@@ -43,7 +43,7 @@ public class BlockingInputChannel implements BoundaryInputChannel {
 
 	private Buffer extraBuffer;
 
-	private final TCPConnectionProvider conProvider;
+	private final ConnectionProvider conProvider;
 
 	private final ConnectionInfo conInfo;
 
@@ -61,13 +61,13 @@ public class BlockingInputChannel implements BoundaryInputChannel {
 
 	private ImmutableList<Object> unProcessedData;
 
-	public BlockingInputChannel(int bufSize, TCPConnectionProvider conProvider,
+	public BlockingInputChannel(int bufSize, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
 				bufferTokenName, debugLevel);
 	}
 
-	public BlockingInputChannel(Buffer buffer, TCPConnectionProvider conProvider,
+	public BlockingInputChannel(Buffer buffer, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index 87136867..0ba71928 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -12,7 +12,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 
 /**
  * This is {@link BoundaryOutputChannel} over TCP. Reads data from the given
@@ -34,7 +34,7 @@ public class BlockingOutputChannel implements BoundaryOutputChannel {
 
 	private final Buffer buffer;
 
-	private final TCPConnectionProvider conProvider;
+	private final ConnectionProvider conProvider;
 
 	private final ConnectionInfo conInfo;
 
@@ -50,13 +50,13 @@ public class BlockingOutputChannel implements BoundaryOutputChannel {
 
 	protected ImmutableList<Object> unProcessedData;
 
-	public BlockingOutputChannel(int bufSize, TCPConnectionProvider conProvider,
+	public BlockingOutputChannel(int bufSize, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this(new ConcurrentArrayBuffer(bufSize), conProvider, conInfo,
 				bufferTokenName, debugLevel);
 	}
 
-	public BlockingOutputChannel(Buffer buffer, TCPConnectionProvider conProvider,
+	public BlockingOutputChannel(Buffer buffer, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 		this.buffer = buffer;
 		this.conProvider = conProvider;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 8db6d5b6..8ba67ef2 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -34,7 +34,7 @@
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -49,7 +49,7 @@ public class CfgStringProcessorImpl implements ConfigurationStringProcessor {
 
 	private Configuration staticConfig = null;
 
-	private TCPConnectionProvider conProvider;
+	private ConnectionProvider conProvider;
 
 	public CfgStringProcessorImpl(StreamNode streamNode) {
 		this.streamNode = streamNode;
@@ -67,7 +67,7 @@ public void process(String json, ConfigType type, DrainData drainData) {
 
 				NetworkInfo networkInfo = new NetworkInfo(iNetAddressMap);
 
-				this.conProvider = new TCPConnectionProvider(
+				this.conProvider = new ConnectionProvider(
 						streamNode.getNodeID(), networkInfo);
 			} else
 				System.err
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 9de2f11d..c399c287 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -15,7 +15,7 @@
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 
@@ -31,7 +31,7 @@
  */
 public class Controller {
 
-	private TCPConnectionProvider conProvider;
+	private ConnectionProvider conProvider;
 
 	private CommunicationManager comManager;
 
@@ -116,7 +116,7 @@ public void newApp(Configuration staticCfg) {
 		inetMap.put(controllerNodeID, comManager.getLocalAddress());
 		builder.putExtraData(GlobalConstants.INETADDRESS_MAP, inetMap);
 		NetworkInfo networkinfo = new NetworkInfo(inetMap);
-		this.conProvider = new TCPConnectionProvider(controllerNodeID,
+		this.conProvider = new ConnectionProvider(controllerNodeID,
 				networkinfo);
 		ConfigurationString json = new ConfigurationString(builder.build()
 				.toJson(), ConfigType.STATIC, null);
@@ -148,7 +148,7 @@ public void sendToAll(Object object) {
 		}
 	}
 
-	public TCPConnectionProvider getConProvider() {
+	public ConnectionProvider getConProvider() {
 		return conProvider;
 	}
 

From e1cf5e8174bfa0d4ee1a448ac516227cc405eda5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 07:24:15 +0800
Subject: [PATCH 311/881] ConnectionProvider has been moved into
 Connection.java

ConnectionProvider has been moved in to Connection.java from TCPConnection.java.
---
 .../impl/distributed/HeadChannel.java         |  2 +-
 .../impl/distributed/TailChannel.java         |  2 +-
 .../common/AsyncTCPConnection.java            |  3 +-
 .../common/BoundaryChannelFactory.java        |  2 +-
 .../impl/distributed/common/Connection.java   | 87 ++++++++++++++++++-
 .../distributed/common/TCPConnection.java     | 87 +------------------
 .../distributed/node/AsyncOutputChannel.java  |  2 +-
 .../distributed/node/BlobsManagerImpl.java    |  2 +-
 .../node/BlockingInputChannel.java            |  2 +-
 .../node/BlockingOutputChannel.java           |  2 +-
 .../node/CfgStringProcessorImpl.java          |  2 +-
 .../impl/distributed/runtimer/Controller.java |  2 +-
 12 files changed, 97 insertions(+), 98 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index ee7a5a06..df1049ab 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -6,7 +6,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 38a23aad..ed70f44b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -11,8 +11,8 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
 public class TailChannel extends BlockingInputChannel {
diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index 91f27cd6..a1235bc8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -17,10 +17,9 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
-import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
  * Uses {@link AsynchronousSocketChannel} from Java's NIO.2 to send data. This
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
index 0da08c9d..28a282d5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelFactory.java
@@ -5,7 +5,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Connection.java b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
index ad670e14..3fcfd176 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Connection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Connection.java
@@ -1,13 +1,18 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.Serializable;
+import java.net.SocketTimeoutException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
@@ -273,4 +278,84 @@ public enum ConnectionType {
 		 * NBIB
 		 */
 	}
+
+	/**
+	 * Keeps all opened {@link TCPConnection}s for a machine. Each machine
+	 * should have a single instance of this class and use this class to make
+	 * new connections.
+	 * 
+	 * <p>
+	 * TODO: Need to make this class singleton. I didn't do it now because in
+	 * current way, controller and a local {@link StreamNode} are running in a
+	 * same JVM. So first, local {@link StreamNode} should be made to run on a
+	 * different JVM and then make this class singleton.
+	 */
+	public static class ConnectionProvider {
+
+		private ConcurrentMap<ConnectionInfo, Connection> allConnections;
+
+		private final int myNodeID;
+
+		private final NetworkInfo networkInfo;
+
+		public ConnectionProvider(int myNodeID, NetworkInfo networkInfo) {
+			checkNotNull(networkInfo, "networkInfo is null");
+			this.myNodeID = myNodeID;
+			this.networkInfo = networkInfo;
+			this.allConnections = new ConcurrentHashMap<>();
+		}
+
+		/**
+		 * See {@link #getConnection(TCPConnectionInfo, int)}.
+		 * 
+		 * @param conInfo
+		 * @return
+		 * @throws IOException
+		 */
+		public Connection getConnection(ConnectionInfo conInfo)
+				throws IOException {
+			return getConnection(conInfo, 0);
+		}
+
+/**
+		 * If the connection corresponds to conInfo is already established
+		 * returns the connection. Try to make a new connection otherwise.
+		 *
+		 * @param conInfo - Information that uniquely identifies a {@link TCPConnection
+		 * @param timeOut - Time out only valid if making connection needs to be
+		 * 			done through a listener socket. i.e, conInfo.getSrcID() == myNodeID.
+		 * @return
+		 * @throws SocketTimeoutException
+		 * @throws IOException
+		 */
+		public Connection getConnection(ConnectionInfo conInfo, int timeOut)
+				throws SocketTimeoutException, IOException {
+			Connection con = allConnections.get(conInfo);
+			if (con != null) {
+				if (con.isStillConnected()) {
+					return con;
+				} else {
+					throw new AssertionError("con.closeConnection()");
+					// con.closeConnection();
+				}
+			}
+
+			con = conInfo.makeConnection(myNodeID, networkInfo, timeOut);
+			if (con == null)
+				throw new IOException("Connection making process failed.");
+
+			allConnections.put(conInfo, con);
+			return con;
+		}
+
+		public void closeAllConnections() {
+			for (Connection con : allConnections.values()) {
+				try {
+					con.closeConnection();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
index 07b31a83..5a3ae9cd 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/TCPConnection.java
@@ -1,7 +1,5 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectInputStream;
@@ -9,16 +7,13 @@
 import java.io.OptionalDataException;
 import java.net.InetAddress;
 import java.net.Socket;
-import java.net.SocketTimeoutException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 import edu.mit.streamjit.impl.distributed.node.BlockingOutputChannel;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
 /**
  * TCPConnection is not thread safe.
@@ -269,84 +264,4 @@ public BoundaryOutputChannel outputChannel(Token t, int bufSize,
 					t.toString(), 0);
 		}
 	}
-
-	/**
-	 * Keeps all opened {@link TCPConnection}s for a machine. Each machine
-	 * should have a single instance of this class and use this class to make
-	 * new connections.
-	 * 
-	 * <p>
-	 * TODO: Need to make this class singleton. I didn't do it now because in
-	 * current way, controller and a local {@link StreamNode} are running in a
-	 * same JVM. So first, local {@link StreamNode} should be made to run on a
-	 * different JVM and then make this class singleton.
-	 */
-	public static class ConnectionProvider {
-
-		private ConcurrentMap<ConnectionInfo, Connection> allConnections;
-
-		private final int myNodeID;
-
-		private final NetworkInfo networkInfo;
-
-		public ConnectionProvider(int myNodeID, NetworkInfo networkInfo) {
-			checkNotNull(networkInfo, "networkInfo is null");
-			this.myNodeID = myNodeID;
-			this.networkInfo = networkInfo;
-			this.allConnections = new ConcurrentHashMap<>();
-		}
-
-		/**
-		 * See {@link #getConnection(TCPConnectionInfo, int)}.
-		 * 
-		 * @param conInfo
-		 * @return
-		 * @throws IOException
-		 */
-		public Connection getConnection(ConnectionInfo conInfo)
-				throws IOException {
-			return getConnection(conInfo, 0);
-		}
-
-/**
-		 * If the connection corresponds to conInfo is already established
-		 * returns the connection. Try to make a new connection otherwise.
-		 *
-		 * @param conInfo - Information that uniquely identifies a {@link TCPConnection
-		 * @param timeOut - Time out only valid if making connection needs to be
-		 * 			done through a listener socket. i.e, conInfo.getSrcID() == myNodeID.
-		 * @return
-		 * @throws SocketTimeoutException
-		 * @throws IOException
-		 */
-		public Connection getConnection(ConnectionInfo conInfo, int timeOut)
-				throws SocketTimeoutException, IOException {
-			Connection con = allConnections.get(conInfo);
-			if (con != null) {
-				if (con.isStillConnected()) {
-					return con;
-				} else {
-					throw new AssertionError("con.closeConnection()");
-					// con.closeConnection();
-				}
-			}
-
-			con = conInfo.makeConnection(myNodeID, networkInfo, timeOut);
-			if (con == null)
-				throw new IOException("Connection making process failed.");
-
-			allConnections.put(conInfo, con);
-			return con;
-		}
-
-		public void closeAllConnections() {
-			for (Connection con : allConnections.values()) {
-				try {
-					con.closeConnection();
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-	}
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
index 3f5f3328..cf734983 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
@@ -10,7 +10,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 
 public class AsyncOutputChannel implements BoundaryOutputChannel {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index d9772120..ac4d4409 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -35,11 +35,11 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 39c25b8e..b2c71542 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 
 /**
  * This is {@link BoundaryInputChannel} over TCP. Receive objects from TCP
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index 0ba71928..bda61f11 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -12,7 +12,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 
 /**
  * This is {@link BoundaryOutputChannel} over TCP. Reads data from the given
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
index 8ba67ef2..1167bb84 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
@@ -31,10 +31,10 @@
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index c399c287..3116af30 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -11,11 +11,11 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
-import edu.mit.streamjit.impl.distributed.common.TCPConnection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 

From 5aee4a0fe9674c064aaba3039a2b2537edb55999 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 08:04:49 +0800
Subject: [PATCH 312/881] 
 Renamed:ConfigurationStringProcessor->ConfigurationProcessor

Renamed:ConfigurationStringProcessor->ConfigurationProcessor
and CfgStringProcessorImpl->ConfigurationProcessorImpl
---
 .../streamjit/impl/distributed/StreamJitAppManager.java   | 2 +-
 .../impl/distributed/common/ConfigurationString.java      | 6 +++---
 .../impl/distributed/node/CTRLRMessageVisitorImpl.java    | 6 +++---
 ...ProcessorImpl.java => ConfigurationProcessorImpl.java} | 8 ++++----
 .../streamjit/impl/distributed/runtimer/Controller.java   | 2 +-
 5 files changed, 12 insertions(+), 12 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/node/{CfgStringProcessorImpl.java => ConfigurationProcessorImpl.java} (97%)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index b1b6cbbd..df8bfd08 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -26,7 +26,7 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Command;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Error.ErrorProcessor;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/ConfigurationString.java b/src/edu/mit/streamjit/impl/distributed/common/ConfigurationString.java
index 4a1fd131..43906551 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/ConfigurationString.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/ConfigurationString.java
@@ -2,7 +2,7 @@
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
@@ -34,7 +34,7 @@ public void accept(CTRLRMessageVisitor visitor) {
 		visitor.visit(this);
 	}
 
-	public void process(ConfigurationStringProcessor jp) {
+	public void process(ConfigurationProcessor jp) {
 		jp.process(jsonString, type, drainData);
 	}
 
@@ -45,7 +45,7 @@ public void process(ConfigurationStringProcessor jp) {
 	 * @author Sumanan sumanan@mit.edu
 	 * @since May 27, 2013
 	 */
-	public interface ConfigurationStringProcessor {
+	public interface ConfigurationProcessor {
 
 		public void process(String cfg, ConfigType type, DrainData drainData);
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index f9d49f94..78a44a04 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.impl.distributed.common.Command;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.MiscCtrlElementProcessor;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
@@ -24,13 +24,13 @@ public class CTRLRMessageVisitorImpl implements CTRLRMessageVisitor {
 
 	private final StreamNode streamNode;
 	private final RequestProcessor rp;
-	private final ConfigurationStringProcessor jp;
+	private final ConfigurationProcessor jp;
 	private final MiscCtrlElementProcessor miscProcessor;
 
 	public CTRLRMessageVisitorImpl(StreamNode streamNode) {
 		this.streamNode = streamNode;
 		this.rp = new RequestProcessorImpl();
-		this.jp = new CfgStringProcessorImpl(streamNode);
+		this.jp = new ConfigurationProcessorImpl(streamNode);
 		this.miscProcessor = new MiscCtrlElementProcessorImpl();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
similarity index 97%
rename from src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
rename to src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 1167bb84..95728734 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CfgStringProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -29,7 +29,7 @@
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter.BlobSpecifier;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Error;
@@ -38,12 +38,12 @@
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
- * {@link ConfigurationStringProcessor} at {@link StreamNode} side.
+ * {@link ConfigurationProcessor} at {@link StreamNode} side.
  * 
  * @author Sumanan sumanan@mit.edu
  * @since May 27, 2013
  */
-public class CfgStringProcessorImpl implements ConfigurationStringProcessor {
+public class ConfigurationProcessorImpl implements ConfigurationProcessor {
 
 	private StreamNode streamNode;
 
@@ -51,7 +51,7 @@ public class CfgStringProcessorImpl implements ConfigurationStringProcessor {
 
 	private ConnectionProvider conProvider;
 
-	public CfgStringProcessorImpl(StreamNode streamNode) {
+	public ConfigurationProcessorImpl(StreamNode streamNode) {
 		this.streamNode = streamNode;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
index 3116af30..a5f434c5 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/Controller.java
@@ -10,7 +10,7 @@
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
-import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationStringProcessor.ConfigType;
+import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;

From 33766491c581d2ae68f43d425a246507f3533bbe Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 10:41:08 +0800
Subject: [PATCH 313/881] Variable conInfoMap has been made final.

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index ac4d4409..8168b329 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -55,7 +55,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private Map<Token, BlobExecuter> blobExecuters;
 	private final StreamNode streamNode;
-	private Map<Token, ConnectionInfo> conInfoMap;
+	private final Map<Token, ConnectionInfo> conInfoMap;
 
 	private MonitorBuffers monBufs;
 

From 3a12bda93bb99ec495305e397186878cd6150ef3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 10:43:23 +0800
Subject: [PATCH 314/881] assert statement has been added at createBE()

assert bufferManager.isbufferSizesReady().
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 8168b329..e208ab0e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -95,6 +95,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 	}
 
 	private void createBEs(ImmutableSet<Blob> blobSet) {
+		assert bufferManager.isbufferSizesReady() : "Buffer sizes must be available to create BlobExecuters.";
 		blobExecuters = new HashMap<>();
 		Set<Token> locaTokens = bufferManager.localTokens();
 		ImmutableMap<Token, Integer> bufferSizesMap = bufferManager

From 4b0c1273c385d6de2986865c032866414bd0bee7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 11:11:16 +0800
Subject: [PATCH 315/881] Comment has been added to the class MonitorBuffers.

MonitorBuffers class profiles the buffer sizes in a timely manner and log that
information into a text file. This information may be useful to analyse and
find out deadlock situations.
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e208ab0e..68e1fc71 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -715,6 +715,14 @@ public void requestStop() {
 
 	private static int count = 0;
 
+	/**
+	 * Profiles the buffer sizes in a timely manner and log that information
+	 * into a text file. This information may be useful to analyse and find out
+	 * deadlock situations.
+	 * 
+	 * @author sumanan
+	 * 
+	 */
 	private class MonitorBuffers extends Thread {
 		private final int id;
 		private final AtomicBoolean stopFlag;

From 807fc03095d1c63bf2d43c35d571d5515022ca50 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 10 Sep 2014 12:30:19 +0800
Subject: [PATCH 316/881] Comment has been added to DrainDeadLockHandler.

---
 .../distributed/node/BlobsManagerImpl.java    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 68e1fc71..4de94ac5 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -188,6 +188,9 @@ private class BlobExecuter {
 
 		private boolean reqDrainData;
 
+		/**
+		 * Buffers for all input and output edges of the {@link #blob}.
+		 */
 		private ImmutableMap<Token, Buffer> bufferMap;
 
 		private BlobExecuter(Token t, Blob blob,
@@ -793,6 +796,23 @@ public void stopMonitoring() {
 		}
 	}
 
+	/**
+	 * Handles another type of deadlock which occurs when draining. A Down blob,
+	 * that has more than one upper blob, cannot progress because some of its
+	 * upper blobs are drained and hence no input on the corresponding input
+	 * channels, and other upper blobs blocked at their output channels as the
+	 * down blob is no more consuming data. So those non-drained upper blobs are
+	 * going to stuck forever at their output channels and the down blob will
+	 * not receive DODrain command from the controller.
+	 * 
+	 * This class just discard the buffer contents so that blocked blobs can
+	 * progress.
+	 * 
+	 * See the Deadlock 5.
+	 * 
+	 * @author sumanan
+	 * 
+	 */
 	private class DrainDeadLockHandler extends Thread {
 		final AtomicBoolean run;
 

From 74b30eeea5b4e070eca41d026b4a2129092bf68b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 13 Sep 2014 06:46:02 +0800
Subject: [PATCH 317/881] Members sorted.

---
 .../distributed/node/BlobsManagerImpl.java    | 664 +++++++++---------
 1 file changed, 339 insertions(+), 325 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 4de94ac5..24a120d7 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -54,20 +54,22 @@
 public class BlobsManagerImpl implements BlobsManager {
 
 	private Map<Token, BlobExecuter> blobExecuters;
-	private final StreamNode streamNode;
+
+	private final BufferManager bufferManager;
+
+	private final CommandProcessor cmdProcessor;
+
 	private final Map<Token, ConnectionInfo> conInfoMap;
 
-	private MonitorBuffers monBufs;
+	private final ConnectionProvider conProvider;
 
 	private volatile DrainDeadLockHandler drainDeadLockHandler;
 
 	private final CTRLRDrainProcessor drainProcessor;
 
-	private final CommandProcessor cmdProcessor;
-
-	private final BufferManager bufferManager;
+	private MonitorBuffers monBufs;
 
-	private final ConnectionProvider conProvider;
+	private final StreamNode streamNode;
 
 	/**
 	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
@@ -94,21 +96,44 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			createBEs(blobSet);
 	}
 
-	private void createBEs(ImmutableSet<Blob> blobSet) {
-		assert bufferManager.isbufferSizesReady() : "Buffer sizes must be available to create BlobExecuters.";
-		blobExecuters = new HashMap<>();
-		Set<Token> locaTokens = bufferManager.localTokens();
-		ImmutableMap<Token, Integer> bufferSizesMap = bufferManager
-				.bufferSizes();
-		for (Blob b : blobSet) {
-			Token t = Utils.getBlobID(b);
-			ImmutableMap<Token, BoundaryInputChannel> inputChannels = createInputChannels(
-					Sets.difference(b.getInputs(), locaTokens), bufferSizesMap);
-			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = createOutputChannels(
-					Sets.difference(b.getOutputs(), locaTokens), bufferSizesMap);
-			blobExecuters.put(t, new BlobExecuter(t, b, inputChannels,
-					outputChannels));
+	/**
+	 * Drain the blob identified by the token.
+	 */
+	public void drain(Token blobID, boolean reqDrainData) {
+		for (BlobExecuter be : blobExecuters.values()) {
+			if (be.getBlobID().equals(blobID)) {
+				be.doDrain(reqDrainData);
+				return;
+			}
 		}
+		throw new IllegalArgumentException(String.format(
+				"No blob with blobID %s", blobID));
+	}
+
+	public CommandProcessor getCommandProcessor() {
+		return cmdProcessor;
+	}
+
+	public CTRLRDrainProcessor getDrainProcessor() {
+		return drainProcessor;
+	}
+
+	public void reqDrainedData(Set<Token> blobSet) {
+		// ImmutableMap.Builder<Token, DrainData> builder = new
+		// ImmutableMap.Builder<>();
+		// for (BlobExecuter be : blobExecuters) {
+		// if (be.isDrained) {
+		// builder.put(be.blobID, be.blob.getDrainData());
+		// }
+		// }
+		//
+		// try {
+		// streamNode.controllerConnection
+		// .writeObject(new SNDrainElement.DrainedData(builder.build()));
+		// } catch (IOException e) {
+		// // TODO Auto-generated catch block
+		// e.printStackTrace();
+		// }
 	}
 
 	/**
@@ -146,6 +171,23 @@ public void stop() {
 			drainDeadLockHandler.stopit();
 	}
 
+	private void createBEs(ImmutableSet<Blob> blobSet) {
+		assert bufferManager.isbufferSizesReady() : "Buffer sizes must be available to create BlobExecuters.";
+		blobExecuters = new HashMap<>();
+		Set<Token> locaTokens = bufferManager.localTokens();
+		ImmutableMap<Token, Integer> bufferSizesMap = bufferManager
+				.bufferSizes();
+		for (Blob b : blobSet) {
+			Token t = Utils.getBlobID(b);
+			ImmutableMap<Token, BoundaryInputChannel> inputChannels = createInputChannels(
+					Sets.difference(b.getInputs(), locaTokens), bufferSizesMap);
+			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = createOutputChannels(
+					Sets.difference(b.getOutputs(), locaTokens), bufferSizesMap);
+			blobExecuters.put(t, new BlobExecuter(t, b, inputChannels,
+					outputChannels));
+		}
+	}
+
 	private ImmutableMap<Token, BoundaryInputChannel> createInputChannels(
 			Set<Token> inputTokens, ImmutableMap<Token, Integer> bufferMap) {
 		ImmutableMap.Builder<Token, BoundaryInputChannel> inputChannelMap = new ImmutableMap.Builder<>();
@@ -168,8 +210,60 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 		return outputChannelMap.build();
 	}
 
+	/**
+	 * Just to added for debugging purpose.
+	 */
+	private synchronized void printDrainedStatus() {
+		System.out.println("****************************************");
+		for (BlobExecuter be : blobExecuters.values()) {
+			switch (be.drainState) {
+				case 0 :
+					System.out.println(String.format("%s - No Drain Called",
+							be.blobID));
+					break;
+				case 1 :
+					System.out.println(String.format("%s - Drain Called",
+							be.blobID));
+					break;
+				case 2 :
+					System.out.println(String.format(
+							"%s - Drain Passed to Interpreter", be.blobID));
+					break;
+				case 3 :
+					System.out.println(String.format(
+							"%s - Returned from Interpreter", be.blobID));
+					break;
+				case 4 :
+					System.out.println(String.format(
+							"%s - Draining Completed. All threads stopped.",
+							be.blobID));
+					break;
+				case 5 :
+					System.out.println(String.format(
+							"%s - Processing Drain data", be.blobID));
+					break;
+				case 6 :
+					System.out.println(String.format("%s - Draindata sent",
+							be.blobID));
+					break;
+			}
+		}
+		System.out.println("****************************************");
+	}
+
 	private class BlobExecuter {
 
+		private Blob blob;
+
+		private final Token blobID;
+
+		private Set<BlobThread2> blobThreads;
+
+		/**
+		 * Buffers for all input and output edges of the {@link #blob}.
+		 */
+		private ImmutableMap<Token, Buffer> bufferMap;
+
 		/**
 		 * This flag will be set to true if an exception thrown by the core code
 		 * of the {@link Blob}. Any exception occurred in a blob's corecode will
@@ -177,22 +271,15 @@ private class BlobExecuter {
 		 * {@link BlobThread2}.
 		 */
 		private AtomicBoolean crashed;
-		private volatile int drainState;
-		private final Token blobID;
 
-		private Blob blob;
-		private Set<BlobThread2> blobThreads;
+		private volatile int drainState;
 
 		private final BoundaryInputChannelManager inChnlManager;
+
 		private final BoundaryOutputChannelManager outChnlManager;
 
 		private boolean reqDrainData;
 
-		/**
-		 * Buffers for all input and output edges of the {@link #blob}.
-		 */
-		private ImmutableMap<Token, Buffer> bufferMap;
-
 		private BlobExecuter(Token t, Blob blob,
 				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
 				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
@@ -218,59 +305,46 @@ private BlobExecuter(Token t, Blob blob,
 			this.blobID = t;
 		}
 
-		/**
-		 * Returns a name for thread.
-		 * 
-		 * @param blob
-		 * @return
-		 */
-		private String getName(Blob blob) {
-			StringBuilder sb = new StringBuilder("Workers-");
-			int limit = 0;
-			for (Worker<?, ?> w : blob.getWorkers()) {
-				sb.append(Workers.getIdentifier(w));
-				sb.append(",");
-				if (++limit > 5)
-					break;
-			}
-			return sb.toString();
-		}
-
-		private void startChannels() {
-			outChnlManager.start();
-			inChnlManager.start();
-		}
-
-		private void start() {
-			outChnlManager.waitToStart();
-			inChnlManager.waitToStart();
-
-			bufferMap = buildBufferMap();
-			blob.installBuffers(bufferMap);
-
-			for (Thread t : blobThreads)
-				t.start();
-
-			System.out.println(blobID + " started");
+		public Token getBlobID() {
+			return blobID;
 		}
 
-		private void stop() {
-			inChnlManager.stop(1);
-			outChnlManager.stop(true);
+		private ImmutableMap<Token, Buffer> buildBufferMap() {
+			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+					.builder();
+			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
+					.localBufferMap();
+			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
+					.inputChannelsMap();
+			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
+					.outputChannelsMap();
 
-			for (Thread t : blobThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
+			for (Token t : blob.getInputs()) {
+				if (localBufferMap.containsKey(t)) {
+					assert !inputChannels.containsKey(t) : "Same channels is exists in both localBuffer and inputChannel";
+					bufferMapBuilder.put(t, localBufferMap.get(t));
+				} else if (inputChannels.containsKey(t)) {
+					BoundaryInputChannel chnl = inputChannels.get(t);
+					bufferMapBuilder.put(t, chnl.getBuffer());
+				} else {
+					throw new AssertionError(String.format(
+							"No Buffer for input channel %s ", t));
 				}
 			}
 
-			inChnlManager.waitToStop();
-			outChnlManager.waitToStop();
-
-			if (monBufs != null)
-				monBufs.stopMonitoring();
+			for (Token t : blob.getOutputs()) {
+				if (localBufferMap.containsKey(t)) {
+					assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
+					bufferMapBuilder.put(t, localBufferMap.get(t));
+				} else if (outputChannels.containsKey(t)) {
+					BoundaryOutputChannel chnl = outputChannels.get(t);
+					bufferMapBuilder.put(t, chnl.getBuffer());
+				} else {
+					throw new AssertionError(String.format(
+							"No Buffer for output channel %s ", t));
+				}
+			}
+			return bufferMapBuilder.build();
 		}
 
 		private void doDrain(boolean reqDrainData) {
@@ -465,170 +539,107 @@ private DrainedData getEmptyDrainData() {
 					inputDataBuilder.build(), outputDataBuilder.build());
 		}
 
-		public Token getBlobID() {
-			return blobID;
+		/**
+		 * Returns a name for thread.
+		 * 
+		 * @param blob
+		 * @return
+		 */
+		private String getName(Blob blob) {
+			StringBuilder sb = new StringBuilder("Workers-");
+			int limit = 0;
+			for (Worker<?, ?> w : blob.getWorkers()) {
+				sb.append(Workers.getIdentifier(w));
+				sb.append(",");
+				if (++limit > 5)
+					break;
+			}
+			return sb.toString();
 		}
 
-		private ImmutableMap<Token, Buffer> buildBufferMap() {
-			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-					.builder();
-			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
-					.localBufferMap();
-			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
-					.inputChannelsMap();
-			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
-					.outputChannelsMap();
+		private void start() {
+			outChnlManager.waitToStart();
+			inChnlManager.waitToStart();
 
-			for (Token t : blob.getInputs()) {
-				if (localBufferMap.containsKey(t)) {
-					assert !inputChannels.containsKey(t) : "Same channels is exists in both localBuffer and inputChannel";
-					bufferMapBuilder.put(t, localBufferMap.get(t));
-				} else if (inputChannels.containsKey(t)) {
-					BoundaryInputChannel chnl = inputChannels.get(t);
-					bufferMapBuilder.put(t, chnl.getBuffer());
-				} else {
-					throw new AssertionError(String.format(
-							"No Buffer for input channel %s ", t));
-				}
-			}
-
-			for (Token t : blob.getOutputs()) {
-				if (localBufferMap.containsKey(t)) {
-					assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
-					bufferMapBuilder.put(t, localBufferMap.get(t));
-				} else if (outputChannels.containsKey(t)) {
-					BoundaryOutputChannel chnl = outputChannels.get(t);
-					bufferMapBuilder.put(t, chnl.getBuffer());
-				} else {
-					throw new AssertionError(String.format(
-							"No Buffer for output channel %s ", t));
-				}
-			}
-			return bufferMapBuilder.build();
-		}
-	}
-
-	private static class DrainCallback implements Runnable {
+			bufferMap = buildBufferMap();
+			blob.installBuffers(bufferMap);
 
-		private final BlobExecuter blobExec;
-		// TODO: [2014-03-17] Just to added for checking the drain time. Remove
-		// it later.
-		private final Stopwatch sw;
+			for (Thread t : blobThreads)
+				t.start();
 
-		DrainCallback(BlobExecuter be) {
-			this.blobExec = be;
-			sw = Stopwatch.createStarted();
+			System.out.println(blobID + " started");
 		}
 
-		@Override
-		public void run() {
-			sw.stop();
-			System.out.println("Time taken to drain " + blobExec.blobID
-					+ " is " + sw.elapsed(TimeUnit.MILLISECONDS) + " ms");
-			blobExec.drained();
+		private void startChannels() {
+			outChnlManager.start();
+			inChnlManager.start();
 		}
-	}
 
-	/**
-	 * Drain the blob identified by the token.
-	 */
-	public void drain(Token blobID, boolean reqDrainData) {
-		for (BlobExecuter be : blobExecuters.values()) {
-			if (be.getBlobID().equals(blobID)) {
-				be.doDrain(reqDrainData);
-				return;
-			}
-		}
-		throw new IllegalArgumentException(String.format(
-				"No blob with blobID %s", blobID));
-	}
+		private void stop() {
+			inChnlManager.stop(1);
+			outChnlManager.stop(true);
 
-	/**
-	 * Just to added for debugging purpose.
-	 */
-	private synchronized void printDrainedStatus() {
-		System.out.println("****************************************");
-		for (BlobExecuter be : blobExecuters.values()) {
-			switch (be.drainState) {
-				case 0 :
-					System.out.println(String.format("%s - No Drain Called",
-							be.blobID));
-					break;
-				case 1 :
-					System.out.println(String.format("%s - Drain Called",
-							be.blobID));
-					break;
-				case 2 :
-					System.out.println(String.format(
-							"%s - Drain Passed to Interpreter", be.blobID));
-					break;
-				case 3 :
-					System.out.println(String.format(
-							"%s - Returned from Interpreter", be.blobID));
-					break;
-				case 4 :
-					System.out.println(String.format(
-							"%s - Draining Completed. All threads stopped.",
-							be.blobID));
-					break;
-				case 5 :
-					System.out.println(String.format(
-							"%s - Processing Drain data", be.blobID));
-					break;
-				case 6 :
-					System.out.println(String.format("%s - Draindata sent",
-							be.blobID));
-					break;
+			for (Thread t : blobThreads) {
+				try {
+					t.join();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
 			}
+
+			inChnlManager.waitToStop();
+			outChnlManager.waitToStop();
+
+			if (monBufs != null)
+				monBufs.stopMonitoring();
 		}
-		System.out.println("****************************************");
 	}
 
-	public void reqDrainedData(Set<Token> blobSet) {
-		// ImmutableMap.Builder<Token, DrainData> builder = new
-		// ImmutableMap.Builder<>();
-		// for (BlobExecuter be : blobExecuters) {
-		// if (be.isDrained) {
-		// builder.put(be.blobID, be.blob.getDrainData());
-		// }
-		// }
-		//
-		// try {
-		// streamNode.controllerConnection
-		// .writeObject(new SNDrainElement.DrainedData(builder.build()));
-		// } catch (IOException e) {
-		// // TODO Auto-generated catch block
-		// e.printStackTrace();
-		// }
-	}
+	private final class BlobThread2 extends Thread {
 
-	public CTRLRDrainProcessor getDrainProcessor() {
-		return drainProcessor;
-	}
+		private final BlobExecuter be;
 
-	public CommandProcessor getCommandProcessor() {
-		return cmdProcessor;
-	}
+		private final Runnable coreCode;
 
-	/**
-	 * Implementation of {@link DrainProcessor} at {@link StreamNode} side. All
-	 * appropriate response logic to successfully perform the draining is
-	 * implemented here.
-	 * 
-	 * @author Sumanan sumanan@mit.edu
-	 * @since Jul 30, 2013
-	 */
-	private class CTRLRDrainProcessorImpl implements CTRLRDrainProcessor {
+		private volatile boolean stopping = false;
 
-		@Override
-		public void process(DrainDataRequest drnDataReq) {
-			System.err.println("Not expected in current situation");
-			// reqDrainedData(drnDataReq.blobsSet);
+		private BlobThread2(Runnable coreCode, BlobExecuter be) {
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		private BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
+			super(name);
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		public void requestStop() {
+			stopping = true;
 		}
 
 		@Override
-		public void process(DoDrain drain) {
-			drain(drain.blobID, drain.reqDrainData);
+		public void run() {
+			try {
+				while (!stopping)
+					coreCode.run();
+			} catch (Error | Exception e) {
+				System.out.println(Thread.currentThread().getName()
+						+ " crashed...");
+				if (be.crashed.compareAndSet(false, true)) {
+					e.printStackTrace();
+					if (be.drainState == 1 || be.drainState == 2)
+						be.drained();
+					else if (be.drainState == 0) {
+						try {
+							streamNode.controllerConnection
+									.writeObject(AppStatus.ERROR);
+						} catch (IOException e1) {
+							e1.printStackTrace();
+						}
+					}
+				}
+			}
 		}
 	}
 
@@ -671,48 +682,126 @@ public void processSTOP() {
 		}
 	}
 
-	private final class BlobThread2 extends Thread {
-		private volatile boolean stopping = false;
-		private final Runnable coreCode;
-		private final BlobExecuter be;
+	/**
+	 * Implementation of {@link DrainProcessor} at {@link StreamNode} side. All
+	 * appropriate response logic to successfully perform the draining is
+	 * implemented here.
+	 * 
+	 * @author Sumanan sumanan@mit.edu
+	 * @since Jul 30, 2013
+	 */
+	private class CTRLRDrainProcessorImpl implements CTRLRDrainProcessor {
 
-		private BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
-			super(name);
-			this.coreCode = coreCode;
-			this.be = be;
+		@Override
+		public void process(DoDrain drain) {
+			drain(drain.blobID, drain.reqDrainData);
 		}
 
-		private BlobThread2(Runnable coreCode, BlobExecuter be) {
-			this.coreCode = coreCode;
-			this.be = be;
+		@Override
+		public void process(DrainDataRequest drnDataReq) {
+			System.err.println("Not expected in current situation");
+			// reqDrainedData(drnDataReq.blobsSet);
+		}
+	}
+
+	private static class DrainCallback implements Runnable {
+
+		private final BlobExecuter blobExec;
+
+		// TODO: [2014-03-17] Just to added for checking the drain time. Remove
+		// it later.
+		private final Stopwatch sw;
+
+		DrainCallback(BlobExecuter be) {
+			this.blobExec = be;
+			sw = Stopwatch.createStarted();
 		}
 
 		@Override
+		public void run() {
+			sw.stop();
+			System.out.println("Time taken to drain " + blobExec.blobID
+					+ " is " + sw.elapsed(TimeUnit.MILLISECONDS) + " ms");
+			blobExec.drained();
+		}
+	}
+
+	/**
+	 * Handles another type of deadlock which occurs when draining. A Down blob,
+	 * that has more than one upper blob, cannot progress because some of its
+	 * upper blobs are drained and hence no input on the corresponding input
+	 * channels, and other upper blobs blocked at their output channels as the
+	 * down blob is no more consuming data. So those non-drained upper blobs are
+	 * going to stuck forever at their output channels and the down blob will
+	 * not receive DODrain command from the controller.
+	 * 
+	 * This class just discard the buffer contents so that blocked blobs can
+	 * progress.
+	 * 
+	 * See the Deadlock 5.
+	 * 
+	 * @author sumanan
+	 * 
+	 */
+	private class DrainDeadLockHandler extends Thread {
+
+		final AtomicBoolean run;
+
+		private DrainDeadLockHandler() {
+			super("DrainDeadLockHandler");
+			this.run = new AtomicBoolean(true);
+		}
+
 		public void run() {
 			try {
-				while (!stopping)
-					coreCode.run();
-			} catch (Error | Exception e) {
-				System.out.println(Thread.currentThread().getName()
-						+ " crashed...");
-				if (be.crashed.compareAndSet(false, true)) {
-					e.printStackTrace();
-					if (be.drainState == 1 || be.drainState == 2)
-						be.drained();
-					else if (be.drainState == 0) {
-						try {
-							streamNode.controllerConnection
-									.writeObject(AppStatus.ERROR);
-						} catch (IOException e1) {
-							e1.printStackTrace();
+				Thread.sleep(60000);
+			} catch (InterruptedException e) {
+				return;
+			}
+
+			System.out
+					.println("DrainDeadLockHandler is goint to clean buffers...");
+			boolean areAllDrained = false;
+
+			while (run.get()) {
+				areAllDrained = true;
+				for (BlobExecuter be : blobExecuters.values()) {
+					if (be.drainState == 1 || be.drainState == 2) {
+						// System.out.println(be.blobID + " is not drained");
+						areAllDrained = false;
+						for (Token t : be.blob.getOutputs()) {
+							Buffer b = be.bufferMap.get(t);
+							int size = b.size();
+							if (size == 0)
+								continue;
+							System.out.println(String.format(
+									"Buffer %s has %d data. Going to clean it",
+									t.toString(), size));
+							Object[] obArray = new Object[size];
+							b.readAll(obArray);
+						}
+						for (Token t : be.blob.getInputs()) {
+							Buffer b = be.bufferMap.get(t);
+							int size = b.size();
+							if (size == 0)
+								continue;
+							System.out.println(String.format(
+									"Buffer %s has %d data. Going to clean it",
+									t.toString(), size));
+							Object[] obArray = new Object[size];
+							b.readAll(obArray);
 						}
 					}
 				}
+
+				if (areAllDrained)
+					break;
 			}
 		}
 
-		public void requestStop() {
-			stopping = true;
+		public void stopit() {
+			this.run.set(false);
+			this.interrupt();
 		}
 	}
 
@@ -727,8 +816,11 @@ public void requestStop() {
 	 * 
 	 */
 	private class MonitorBuffers extends Thread {
+
 		private final int id;
+
 		private final AtomicBoolean stopFlag;
+
 		int sleepTime = 25000;
 
 		MonitorBuffers() {
@@ -795,82 +887,4 @@ public void stopMonitoring() {
 			this.interrupt();
 		}
 	}
-
-	/**
-	 * Handles another type of deadlock which occurs when draining. A Down blob,
-	 * that has more than one upper blob, cannot progress because some of its
-	 * upper blobs are drained and hence no input on the corresponding input
-	 * channels, and other upper blobs blocked at their output channels as the
-	 * down blob is no more consuming data. So those non-drained upper blobs are
-	 * going to stuck forever at their output channels and the down blob will
-	 * not receive DODrain command from the controller.
-	 * 
-	 * This class just discard the buffer contents so that blocked blobs can
-	 * progress.
-	 * 
-	 * See the Deadlock 5.
-	 * 
-	 * @author sumanan
-	 * 
-	 */
-	private class DrainDeadLockHandler extends Thread {
-		final AtomicBoolean run;
-
-		private DrainDeadLockHandler() {
-			super("DrainDeadLockHandler");
-			this.run = new AtomicBoolean(true);
-		}
-
-		public void run() {
-			try {
-				Thread.sleep(60000);
-			} catch (InterruptedException e) {
-				return;
-			}
-
-			System.out
-					.println("DrainDeadLockHandler is goint to clean buffers...");
-			boolean areAllDrained = false;
-
-			while (run.get()) {
-				areAllDrained = true;
-				for (BlobExecuter be : blobExecuters.values()) {
-					if (be.drainState == 1 || be.drainState == 2) {
-						// System.out.println(be.blobID + " is not drained");
-						areAllDrained = false;
-						for (Token t : be.blob.getOutputs()) {
-							Buffer b = be.bufferMap.get(t);
-							int size = b.size();
-							if (size == 0)
-								continue;
-							System.out.println(String.format(
-									"Buffer %s has %d data. Going to clean it",
-									t.toString(), size));
-							Object[] obArray = new Object[size];
-							b.readAll(obArray);
-						}
-						for (Token t : be.blob.getInputs()) {
-							Buffer b = be.bufferMap.get(t);
-							int size = b.size();
-							if (size == 0)
-								continue;
-							System.out.println(String.format(
-									"Buffer %s has %d data. Going to clean it",
-									t.toString(), size));
-							Object[] obArray = new Object[size];
-							b.readAll(obArray);
-						}
-					}
-				}
-
-				if (areAllDrained)
-					break;
-			}
-		}
-
-		public void stopit() {
-			this.run.set(false);
-			this.interrupt();
-		}
-	}
 }

From 8c3287a5cad7f04d0f7fd159bb7802691237426f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 13 Sep 2014 07:45:52 +0800
Subject: [PATCH 318/881] StreamJitAppManager's members sorted.

---
 .../impl/distributed/StreamJitAppManager.java | 406 +++++++++---------
 1 file changed, 203 insertions(+), 203 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index df8bfd08..82a9eee8 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -42,27 +42,23 @@
 
 public class StreamJitAppManager {
 
-	private SNDrainProcessorImpl dp = null;
-
-	private SNExceptionProcessorImpl exP = null;
-
-	private ErrorProcessor ep = null;
+	private final StreamJitApp app;
 
 	private AppStatusProcessorImpl apStsPro = null;
 
-	private final Controller controller;
+	private final ConfigurationManager cfgManager;
 
-	private final ConnectionManager conManager;
+	private Map<Token, ConnectionInfo> conInfoMap;
 
-	private final StreamJitApp app;
+	private final ConnectionManager conManager;
 
-	private final ConfigurationManager cfgManager;
+	private final Controller controller;
 
-	private final Token headToken;
+	private SNDrainProcessorImpl dp = null;
 
-	private final Token tailToken;
+	private ErrorProcessor ep = null;
 
-	private boolean isRunning;
+	private SNExceptionProcessorImpl exP = null;
 
 	/**
 	 * A {@link BoundaryOutputChannel} for the head of the stream graph. If the
@@ -72,6 +68,14 @@ public class StreamJitAppManager {
 	 */
 	private BoundaryOutputChannel headChannel;
 
+	private Thread headThread;
+
+	private final Token headToken;
+
+	private boolean isRunning;
+
+	private volatile AppStatus status;
+
 	/**
 	 * A {@link BoundaryInputChannel} for the tail of the whole stream graph. If
 	 * the sink {@link Worker} happened to fall outside the {@link Controller},
@@ -80,13 +84,14 @@ public class StreamJitAppManager {
 	 */
 	private TailChannel tailChannel;
 
-	private Thread headThread;
-
 	private Thread tailThread;
 
-	private volatile AppStatus status;
+	private final Token tailToken;
 
-	private Map<Token, ConnectionInfo> conInfoMap;
+	/**
+	 * [2014-03-15] Just to measure the draining time
+	 */
+	AtomicReference<Stopwatch> stopwatchRef = new AtomicReference<>();
 
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
 			ConfigurationManager cfgManager, ConnectionManager conManager) {
@@ -109,6 +114,100 @@ public StreamJitAppManager(Controller controller, StreamJitApp app,
 		tailToken = Token.createOverallOutputToken(app.sink);
 	}
 
+	public AppStatusProcessor appStatusProcessor() {
+		return apStsPro;
+	}
+
+	public void drain(Token blobID, boolean isFinal) {
+		// System.out.println("Drain requested to blob " + blobID);
+		if (!app.blobtoMachineMap.containsKey(blobID))
+			throw new IllegalArgumentException(blobID
+					+ " not found in the blobtoMachineMap");
+		int nodeID = app.blobtoMachineMap.get(blobID);
+		controller
+				.send(nodeID, new CTRLRDrainElement.DoDrain(blobID, !isFinal));
+	}
+
+	public void drainingFinished(boolean isFinal) {
+		System.out.println("App Manager : Draining Finished...");
+
+		if (headChannel != null) {
+			try {
+				headThread.join();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
+		if (tailChannel != null) {
+			if (isFinal)
+				tailChannel.stop(1);
+			else if (GlobalConstants.useDrainData)
+				tailChannel.stop(2);
+			else
+				tailChannel.stop(3);
+			try {
+				tailThread.join();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
+		if (isFinal)
+			stop();
+
+		isRunning = false;
+
+		Stopwatch sw = stopwatchRef.get();
+		if (sw != null) {
+			sw.stop();
+			long time = sw.elapsed(TimeUnit.MILLISECONDS);
+			System.out.println("Draining time is " + time + " milli seconds");
+		}
+	}
+
+	public void drainingStarted(boolean isFinal) {
+		stopwatchRef.set(Stopwatch.createStarted());
+		if (headChannel != null) {
+			headChannel.stop(isFinal);
+			// [2014-03-16] Moved to drainingFinished. In any case if headThread
+			// blocked at tcp write, draining will also blocked.
+			// try {
+			// headThread.join();
+			// } catch (InterruptedException e) {
+			// e.printStackTrace();
+			// }
+		}
+	}
+
+	public SNDrainProcessor drainProcessor() {
+		return dp;
+	}
+
+	public ErrorProcessor errorProcessor() {
+		return ep;
+	}
+
+	public SNExceptionProcessor exceptionProcessor() {
+		return exP;
+	}
+
+	public long getFixedOutputTime() throws InterruptedException {
+		long time = tailChannel.getFixedOutputTime();
+		if (apStsPro.error) {
+			return -1l;
+		}
+		return time;
+	}
+
+	public AppStatus getStatus() {
+		return status;
+	}
+
+	public boolean isRunning() {
+		return isRunning;
+	}
+
 	public boolean reconfigure(int multiplier) {
 		reset();
 		Configuration.Builder builder = Configuration.builder(cfgManager
@@ -156,6 +255,23 @@ public boolean reconfigure(int multiplier) {
 		return isRunning;
 	}
 
+	public void setDrainer(AbstractDrainer drainer) {
+		assert dp == null : "SNDrainProcessor has already been set";
+		this.dp = new SNDrainProcessorImpl(drainer);
+	}
+
+	public void stop() {
+		this.status = AppStatus.STOPPED;
+		tailChannel.releaseAll();
+		controller.closeAll();
+		dp.drainer.stop();
+	}
+
+	private void reset() {
+		exP.exConInfos = new HashSet<>();
+		apStsPro.reset();
+	}
+
 	/**
 	 * Setup the headchannel and tailchannel.
 	 * 
@@ -226,120 +342,99 @@ private void start() {
 		}
 	}
 
-	public boolean isRunning() {
-		return isRunning;
-	}
-
 	/**
-	 * [2014-03-15] Just to measure the draining time
+	 * {@link AppStatusProcessor} at {@link Controller} side.
+	 * 
+	 * @author Sumanan sumanan@mit.edu
+	 * @since Aug 11, 2013
 	 */
-	AtomicReference<Stopwatch> stopwatchRef = new AtomicReference<>();
+	private class AppStatusProcessorImpl implements AppStatusProcessor {
 
-	public void drainingStarted(boolean isFinal) {
-		stopwatchRef.set(Stopwatch.createStarted());
-		if (headChannel != null) {
-			headChannel.stop(isFinal);
-			// [2014-03-16] Moved to drainingFinished. In any case if headThread
-			// blocked at tcp write, draining will also blocked.
-			// try {
-			// headThread.join();
-			// } catch (InterruptedException e) {
-			// e.printStackTrace();
-			// }
-		}
-	}
+		private boolean compilationError;
 
-	public void drain(Token blobID, boolean isFinal) {
-		// System.out.println("Drain requested to blob " + blobID);
-		if (!app.blobtoMachineMap.containsKey(blobID))
-			throw new IllegalArgumentException(blobID
-					+ " not found in the blobtoMachineMap");
-		int nodeID = app.blobtoMachineMap.get(blobID);
-		controller
-				.send(nodeID, new CTRLRDrainElement.DoDrain(blobID, !isFinal));
-	}
+		private CountDownLatch compileLatch;
 
-	public void drainingFinished(boolean isFinal) {
-		System.out.println("App Manager : Draining Finished...");
+		private volatile boolean error;
 
-		if (headChannel != null) {
-			try {
-				headThread.join();
-			} catch (InterruptedException e) {
-				e.printStackTrace();
-			}
-		}
+		private final int noOfnodes;
 
-		if (tailChannel != null) {
-			if (isFinal)
-				tailChannel.stop(1);
-			else if (GlobalConstants.useDrainData)
-				tailChannel.stop(2);
-			else
-				tailChannel.stop(3);
-			try {
-				tailThread.join();
-			} catch (InterruptedException e) {
-				e.printStackTrace();
-			}
+		private AppStatusProcessorImpl(int noOfnodes) {
+			this.noOfnodes = noOfnodes;
 		}
 
-		if (isFinal)
-			stop();
+		@Override
+		public void processCOMPILATION_ERROR() {
+			System.err.println("Compilation error");
+			this.compilationError = true;
+			compileLatch.countDown();
+		}
 
-		isRunning = false;
+		@Override
+		public void processCOMPILED() {
+			compileLatch.countDown();
+		}
 
-		Stopwatch sw = stopwatchRef.get();
-		if (sw != null) {
-			sw.stop();
-			long time = sw.elapsed(TimeUnit.MILLISECONDS);
-			System.out.println("Draining time is " + time + " milli seconds");
+		@Override
+		public void processERROR() {
+			this.error = true;
+			// This will release the OpenTuner thread which is waiting for fixed
+			// output.
+			tailChannel.releaseAll();
 		}
-	}
 
-	public long getFixedOutputTime() throws InterruptedException {
-		long time = tailChannel.getFixedOutputTime();
-		if (apStsPro.error) {
-			return -1l;
+		@Override
+		public void processNO_APP() {
 		}
-		return time;
-	}
 
-	public void setDrainer(AbstractDrainer drainer) {
-		assert dp == null : "SNDrainProcessor has already been set";
-		this.dp = new SNDrainProcessorImpl(drainer);
-	}
+		@Override
+		public void processNOT_STARTED() {
+		}
 
-	public SNDrainProcessor drainProcessor() {
-		return dp;
-	}
+		@Override
+		public void processRUNNING() {
+		}
 
-	public SNExceptionProcessor exceptionProcessor() {
-		return exP;
-	}
+		@Override
+		public void processSTOPPED() {
+		}
 
-	public ErrorProcessor errorProcessor() {
-		return ep;
-	}
+		private void reset() {
+			compileLatch = new CountDownLatch(noOfnodes);
+			this.compilationError = false;
+			this.error = false;
+		}
 
-	public AppStatusProcessor appStatusProcessor() {
-		return apStsPro;
+		private boolean waitForCompilation() {
+			try {
+				compileLatch.await();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+			return !this.compilationError;
+		}
 	}
 
-	public AppStatus getStatus() {
-		return status;
-	}
+	/**
+	 * {@link ErrorProcessor} at {@link Controller} side.
+	 * 
+	 * @author Sumanan sumanan@mit.edu
+	 * @since Aug 11, 2013
+	 */
+	private class ErrorProcessorImpl implements ErrorProcessor {
 
-	private void reset() {
-		exP.exConInfos = new HashSet<>();
-		apStsPro.reset();
-	}
+		@Override
+		public void processFILE_NOT_FOUND() {
+			System.err
+					.println("No application jar file in streamNode. Terminating...");
+			stop();
+		}
 
-	public void stop() {
-		this.status = AppStatus.STOPPED;
-		tailChannel.releaseAll();
-		controller.closeAll();
-		dp.drainer.stop();
+		@Override
+		public void processWORKER_NOT_FOUND() {
+			System.err
+					.println("No top level class in the jar file. Terminating...");
+			stop();
+		}
 	}
 
 	/**
@@ -378,10 +473,6 @@ private SNExceptionProcessorImpl() {
 			exConInfos = new HashSet<>();
 		}
 
-		@Override
-		public void process(SNException ex) {
-		}
-
 		@Override
 		public void process(AddressBindException abEx) {
 			synchronized (abExLock) {
@@ -415,100 +506,9 @@ public void process(AddressBindException abEx) {
 				controller.send(coninfo.getDstID(), msg);
 			}
 		}
-	}
-
-	/**
-	 * {@link ErrorProcessor} at {@link Controller} side.
-	 * 
-	 * @author Sumanan sumanan@mit.edu
-	 * @since Aug 11, 2013
-	 */
-	private class ErrorProcessorImpl implements ErrorProcessor {
-
-		@Override
-		public void processFILE_NOT_FOUND() {
-			System.err
-					.println("No application jar file in streamNode. Terminating...");
-			stop();
-		}
 
 		@Override
-		public void processWORKER_NOT_FOUND() {
-			System.err
-					.println("No top level class in the jar file. Terminating...");
-			stop();
-		}
-	}
-
-	/**
-	 * {@link AppStatusProcessor} at {@link Controller} side.
-	 * 
-	 * @author Sumanan sumanan@mit.edu
-	 * @since Aug 11, 2013
-	 */
-	private class AppStatusProcessorImpl implements AppStatusProcessor {
-
-		private CountDownLatch compileLatch;
-
-		private boolean compilationError;
-
-		private volatile boolean error;
-
-		private final int noOfnodes;
-
-		private AppStatusProcessorImpl(int noOfnodes) {
-			this.noOfnodes = noOfnodes;
-		}
-
-		@Override
-		public void processRUNNING() {
-		}
-
-		@Override
-		public void processSTOPPED() {
-		}
-
-		@Override
-		public void processERROR() {
-			this.error = true;
-			// This will release the OpenTuner thread which is waiting for fixed
-			// output.
-			tailChannel.releaseAll();
-		}
-
-		@Override
-		public void processNOT_STARTED() {
-		}
-
-		@Override
-		public void processNO_APP() {
-		}
-
-		@Override
-		public void processCOMPILED() {
-			compileLatch.countDown();
-		}
-
-		@Override
-		public void processCOMPILATION_ERROR() {
-			System.err.println("Compilation error");
-			this.compilationError = true;
-			compileLatch.countDown();
-		}
-
-		private void reset() {
-			compileLatch = new CountDownLatch(noOfnodes);
-			this.compilationError = false;
-			this.error = false;
-		}
-
-		private boolean waitForCompilation() {
-			try {
-				compileLatch.await();
-			} catch (InterruptedException e) {
-				e.printStackTrace();
-			}
-			return !this.compilationError;
+		public void process(SNException ex) {
 		}
 	}
 }

From 9b5c32cec77f7d44104bfe0e28b18ee66fc4dce5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 13 Sep 2014 15:29:24 +0800
Subject: [PATCH 319/881] Comments added to the method buildBufferMap()

BlobsManagerImpl#BlobExecutor.buildBufferMap().
---
 .../distributed/node/BlobsManagerImpl.java     | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 24a120d7..ac254c83 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -20,6 +20,7 @@
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.Buffers;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
@@ -34,6 +35,7 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
+import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
@@ -309,6 +311,22 @@ public Token getBlobID() {
 			return blobID;
 		}
 
+		/**
+		 * Gets buffer from {@link BoundaryChannel}s and builds bufferMap. The
+		 * bufferMap will contain all input and output edges of the
+		 * {@link #blob}.
+		 * 
+		 * Note that, Some {@link BoundaryChannel}s (e.g.,
+		 * {@link AsyncOutputChannel}) create {@link Buffer}s after establishing
+		 * {@link Connection} with other end. So this method must be called
+		 * after establishing all IO connections.
+		 * {@link InputChannelManager#waitToStart()} and
+		 * {@link OutputChannelManager#waitToStart()} ensure that the IO
+		 * connections are successfully established.
+		 * 
+		 * @return Buffer map which contains {@link Buffers} for all input and
+		 *         output edges of the {@link #blob}.
+		 */
 		private ImmutableMap<Token, Buffer> buildBufferMap() {
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
 					.builder();

From e2f46fa0beb03b216ef35486aebd66b5f3f04c72 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 14 Sep 2014 08:53:03 +0800
Subject: [PATCH 320/881] Refactored: Private fileWriter() added.

The code snippet that initializes fileWriter is moved from constructor
to a new private method.
---
 .../impl/distributed/node/BlockingInputChannel.java          | 5 ++++-
 .../impl/distributed/node/BlockingOutputChannel.java         | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index b2c71542..6af29a06 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -80,7 +80,10 @@ public BlockingInputChannel(Buffer buffer, ConnectionProvider conProvider,
 		this.isClosed = false;
 		this.stopType = new AtomicInteger(0);
 		count = 0;
+		writer = fileWriter();
+	}
 
+	private FileWriter fileWriter() {
 		FileWriter w = null;
 		if (this.debugLevel == 5) {
 			try {
@@ -91,7 +94,7 @@ public BlockingInputChannel(Buffer buffer, ConnectionProvider conProvider,
 				e.printStackTrace();
 			}
 		}
-		writer = w;
+		return w;
 	}
 
 	private void closeConnection() throws IOException {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index bda61f11..f1f0b149 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -67,7 +67,10 @@ public BlockingOutputChannel(Buffer buffer, ConnectionProvider conProvider,
 		this.debugLevel = debugLevel;
 		this.unProcessedData = null;
 		count = 0;
+		writer = fileWriter();
+	}
 
+	private FileWriter fileWriter() {
 		FileWriter w = null;
 		if (this.debugLevel == 5) {
 			try {
@@ -78,7 +81,7 @@ public BlockingOutputChannel(Buffer buffer, ConnectionProvider conProvider,
 				e.printStackTrace();
 			}
 		}
-		writer = w;
+		return w;
 	}
 
 	private void closeConnection() throws IOException {

From 2be01a0b493e8f37a29ee6c66420b92a2afaf0e8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 15 Sep 2014 23:04:10 +0800
Subject: [PATCH 321/881] Spelling : createLocaBuffers->createLocalBuffers

---
 .../mit/streamjit/impl/distributed/node/BufferManager.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index 8d979b9b..cb723c12 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -132,7 +132,7 @@ public ImmutableMap<Token, Buffer> localBufferMap() {
 			return localBufferMap;
 		}
 
-		protected final void createLocaBuffers() {
+		protected final void createLocalBuffers() {
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
 					.<Token, Buffer> builder();
 			for (Token t : localTokens) {
@@ -155,7 +155,7 @@ public LocalBufferManager(Set<Blob> blobSet) {
 		@Override
 		public void initialise() {
 			bufferSizes = calculateBufferSizes(blobSet);
-			createLocaBuffers();
+			createLocalBuffers();
 			isbufferSizesReady = true;
 		}
 

From 9dc5b06f361a8c89f0db69432f390aeca133880d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 10 Mar 2014 12:56:36 +0800
Subject: [PATCH 322/881] Dynamic buffer has been added

---
 .../impl/distributed/node/DynamicBuffer.java  | 108 ++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
new file mode 100644
index 00000000..bbd109f5
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
@@ -0,0 +1,108 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.mit.streamjit.impl.blob.Buffer;
+
+/**
+ * Dynamically increases supplied buffer capacity in order to avoid dead locks.
+ * Actually creates a new instance of the supplied buffer and copy the data from
+ * old buffer to new buffer. A decorator pattern for {@link Buffer}.
+ * 
+ * <p>
+ * Determining whether buffer is full due to deadlock situation or the current
+ * blob is executing on a faster node than the down stream blob is little
+ * tricky. Here we assume blobs are running on nearly equivalent
+ * 
+ * @author sumanan
+ * @since Mar 10, 2014
+ * 
+ */
+public class DynamicBuffer implements Buffer {
+	
+	
+	private Buffer buffer;
+	private final List<?> initialArguments;
+	private final int initialCapacity;
+	private final int capacityPos;
+	private final Class<? extends Buffer> bufferClass;
+	private final Constructor<? extends Buffer> cons;
+
+	public DynamicBuffer(Class<? extends Buffer> bufferClass,
+			List<?> initialArguments, int initialCapacity, int capacityPos)
+			throws NoSuchMethodException, SecurityException {
+		this.bufferClass = bufferClass;
+		this.initialArguments = initialArguments;
+		this.initialCapacity = initialCapacity;
+		this.capacityPos = capacityPos;
+		this.cons = bufferClass.getConstructor((Class<?>[]) initialArguments
+				.toArray());
+		this.buffer = getNewBuffer(getArguments(initialCapacity));
+	}
+
+	private List<?> getArguments(int newCapacity) {
+		List<Object> newArgs = new ArrayList<>(initialArguments.size());
+		for (int i = 0; i < initialArguments.size(); i++) {
+			if (i == capacityPos)
+				newArgs.add(newCapacity);
+			else
+				newArgs.add(initialArguments.get(i));
+		}
+		return newArgs;
+	}
+
+	private Buffer getNewBuffer(List<?> args) {
+		Buffer buffer;
+		try {
+			buffer = cons.newInstance(args);
+			return buffer;
+		} catch (InstantiationException | IllegalAccessException
+				| IllegalArgumentException | InvocationTargetException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+	
+	@Override
+	public Object read() {
+		return buffer.read();
+	}
+
+	@Override
+	public int read(Object[] data, int offset, int length) {
+		return buffer.read(data, offset, length);
+	}
+
+	@Override
+	public boolean readAll(Object[] data) {
+		return buffer.readAll(data);
+	}
+
+	@Override
+	public boolean readAll(Object[] data, int offset) {
+		return buffer.readAll(data, offset);
+	}
+
+	@Override
+	public boolean write(Object t) {
+		return buffer.write(t);
+	}
+
+	@Override
+	public int write(Object[] data, int offset, int length) {
+		return buffer.write(data, offset, length);
+	}
+
+	@Override
+	public int size() {
+		return buffer.size();
+	}
+
+	@Override
+	public int capacity() {
+		return buffer.capacity();
+	}
+}

From 67d231932cf5765cdd35ec9f1fc5d7044dcd7bc8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 10 Mar 2014 13:34:44 +0800
Subject: [PATCH 323/881] Bug fix

Handles the exceptions thrown.
---
 .../impl/distributed/node/DynamicBuffer.java  | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
index bbd109f5..5f2300bd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
@@ -6,6 +6,8 @@
 import java.util.List;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.util.ConstructorSupplier;
+import edu.mit.streamjit.util.ReflectionUtils;
 
 /**
  * Dynamically increases supplied buffer capacity in order to avoid dead locks.
@@ -17,13 +19,16 @@
  * blob is executing on a faster node than the down stream blob is little
  * tricky. Here we assume blobs are running on nearly equivalent
  * 
+ * <p>
+ * TODO: {@link ConstructorSupplier} can be reused here to instantiate the
+ * buffer instances if we make {@link ConstructorSupplier}.arguments not final.
+ * 
  * @author sumanan
  * @since Mar 10, 2014
  * 
  */
 public class DynamicBuffer implements Buffer {
-	
-	
+
 	private Buffer buffer;
 	private final List<?> initialArguments;
 	private final int initialCapacity;
@@ -32,14 +37,19 @@ public class DynamicBuffer implements Buffer {
 	private final Constructor<? extends Buffer> cons;
 
 	public DynamicBuffer(Class<? extends Buffer> bufferClass,
-			List<?> initialArguments, int initialCapacity, int capacityPos)
-			throws NoSuchMethodException, SecurityException {
+			List<?> initialArguments, int initialCapacity, int capacityPos) {
 		this.bufferClass = bufferClass;
 		this.initialArguments = initialArguments;
 		this.initialCapacity = initialCapacity;
 		this.capacityPos = capacityPos;
-		this.cons = bufferClass.getConstructor((Class<?>[]) initialArguments
-				.toArray());
+		Constructor<? extends Buffer> con = null;
+		try {
+			con = ReflectionUtils
+					.findConstructor(bufferClass, initialArguments);
+		} catch (NoSuchMethodException e1) {
+			e1.printStackTrace();
+		}
+		this.cons = con;
 		this.buffer = getNewBuffer(getArguments(initialCapacity));
 	}
 
@@ -57,7 +67,7 @@ private List<?> getArguments(int newCapacity) {
 	private Buffer getNewBuffer(List<?> args) {
 		Buffer buffer;
 		try {
-			buffer = cons.newInstance(args);
+			buffer = cons.newInstance(args.toArray());
 			return buffer;
 		} catch (InstantiationException | IllegalAccessException
 				| IllegalArgumentException | InvocationTargetException e) {
@@ -65,7 +75,7 @@ private Buffer getNewBuffer(List<?> args) {
 			return null;
 		}
 	}
-	
+
 	@Override
 	public Object read() {
 		return buffer.read();

From 78f55ef6f084cf90bef67efcff7db2a1c86a9a21 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 10 Mar 2014 14:02:07 +0800
Subject: [PATCH 324/881] Renamed : DynamicBuffer -> DynamicBufferManager

---
 .../node/{DynamicBuffer.java => DynamicBufferManager.java}        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/node/{DynamicBuffer.java => DynamicBufferManager.java} (100%)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
similarity index 100%
rename from src/edu/mit/streamjit/impl/distributed/node/DynamicBuffer.java
rename to src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java

From 11ded6cbaf694eaf9e7e25820d34992af0603a56 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 10 Mar 2014 21:07:03 +0800
Subject: [PATCH 325/881] Full implementation of Dynamic buffer.

---
 .../node/DynamicBufferManager.java            | 308 +++++++++++++-----
 1 file changed, 225 insertions(+), 83 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
index 5f2300bd..a0b13a74 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
@@ -4,115 +4,257 @@
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.util.ConstructorSupplier;
 import edu.mit.streamjit.util.ReflectionUtils;
 
 /**
- * Dynamically increases supplied buffer capacity in order to avoid dead locks.
- * Actually creates a new instance of the supplied buffer and copy the data from
- * old buffer to new buffer. A decorator pattern for {@link Buffer}.
- * 
- * <p>
- * Determining whether buffer is full due to deadlock situation or the current
- * blob is executing on a faster node than the down stream blob is little
- * tricky. Here we assume blobs are running on nearly equivalent
- * 
- * <p>
- * TODO: {@link ConstructorSupplier} can be reused here to instantiate the
- * buffer instances if we make {@link ConstructorSupplier}.arguments not final.
+ * <b> Use a {@link DynamicBufferManager} per blob ( i.e., one to one mapping
+ * between a blob and a DynamicBufferManager). Do not use same instance of
+ * {@link DynamicBufferManager} to create buffers for multiple blobs. </b>
  * 
  * @author sumanan
  * @since Mar 10, 2014
  * 
  */
-public class DynamicBuffer implements Buffer {
+public class DynamicBufferManager {
+
+	/**
+	 * keep track of all buffers created for a particular blob.
+	 */
+	List<Buffer> buffers;
 
-	private Buffer buffer;
-	private final List<?> initialArguments;
-	private final int initialCapacity;
-	private final int capacityPos;
-	private final Class<? extends Buffer> bufferClass;
-	private final Constructor<? extends Buffer> cons;
+	public DynamicBufferManager() {
+		buffers = new ArrayList<>();
+	}
 
-	public DynamicBuffer(Class<? extends Buffer> bufferClass,
+	/**
+	 * Make and return a dynamic buffer backed by an instance of bufferClass.
+	 * 
+	 * @param bufferClass
+	 *            : Any concrete implementation of {@link Buffer}.
+	 * @param initialArguments
+	 *            : Constructor arguments. : Initial capacity of the buffer.
+	 * @param capacityPos
+	 *            : the position of size parameter in the bufferClass.
+	 * @return : A dynamic buffer backed by an instance of bufferClass.
+	 */
+	public Buffer getBuffer(Class<? extends Buffer> bufferClass,
 			List<?> initialArguments, int initialCapacity, int capacityPos) {
-		this.bufferClass = bufferClass;
-		this.initialArguments = initialArguments;
-		this.initialCapacity = initialCapacity;
-		this.capacityPos = capacityPos;
-		Constructor<? extends Buffer> con = null;
-		try {
-			con = ReflectionUtils
-					.findConstructor(bufferClass, initialArguments);
-		} catch (NoSuchMethodException e1) {
-			e1.printStackTrace();
-		}
-		this.cons = con;
-		this.buffer = getNewBuffer(getArguments(initialCapacity));
+		Buffer buf = new DynamicBuffer(bufferClass, initialArguments,
+				initialCapacity, capacityPos);
+		buffers.add(buf);
+		return buf;
 	}
 
-	private List<?> getArguments(int newCapacity) {
-		List<Object> newArgs = new ArrayList<>(initialArguments.size());
-		for (int i = 0; i < initialArguments.size(); i++) {
-			if (i == capacityPos)
-				newArgs.add(newCapacity);
-			else
-				newArgs.add(initialArguments.get(i));
+	/**
+	 * Dynamically increases supplied buffer capacity in order to avoid dead
+	 * locks. Actually creates a new instance of the supplied buffer and copy
+	 * the data from old buffer to new buffer. A decorator pattern for
+	 * {@link Buffer}.
+	 * 
+	 * <p>
+	 * Determining whether buffer is full due to deadlock situation or the
+	 * current blob is executing on a faster node than the down stream blob is
+	 * little tricky. Here we assume blobs are running on nearly equivalent.
+	 * </p>
+	 * 
+	 * <p>
+	 * TODO: {@link ConstructorSupplier} can be reused here to instantiate the
+	 * buffer instances if we make {@link ConstructorSupplier}.arguments not
+	 * final.
+	 * </p>
+	 * 
+	 * <p>
+	 * TODO: Possible performance bug during read() due to volatile buffer
+	 * variable and the need for acquire readLock for every single reading. Any
+	 * way to improve this???
+	 * 
+	 * @author sumanan
+	 * @since Mar 10, 2014
+	 * 
+	 */
+	private class DynamicBuffer implements Buffer {
+
+		/**
+		 * Minimum time gap between the last successful write and the current
+		 * time in order to consider the option of doubling the buffer
+		 */
+		private final long gap;
+
+		/**
+		 * Every successful write operation should update this time.
+		 */
+		private long lastWrittenTime;
+
+		/**
+		 * When the algorithm detects there are some progress ( May be after
+		 * some expansions), this flag is set to stop any future expansions.
+		 * This is to prevent infinity buffer growth.
+		 */
+		private boolean expandable;
+
+		/**
+		 * Read lock should be acquired at every single read where as write lock
+		 * will be acquired only when switching the buffer from old to new.
+		 */
+		private ReadWriteLock rwlock;
+
+		private final Class<? extends Buffer> bufferClass;
+		private final List<?> initialArguments;
+		private final int initialCapacity;
+		private final int capacityPos;
+		private final Constructor<? extends Buffer> cons;
+
+		/**
+		 * TODO: Volatileness may severely affects the reading performance.
+		 */
+		private volatile Buffer buffer;
+
+		public DynamicBuffer(Class<? extends Buffer> bufferClass,
+				List<?> initialArguments, int initialCapacity, int capacityPos) {
+			this.bufferClass = bufferClass;
+			this.initialArguments = initialArguments;
+			this.initialCapacity = initialCapacity;
+			this.capacityPos = capacityPos;
+			Constructor<? extends Buffer> con = null;
+			try {
+				con = ReflectionUtils.findConstructor(bufferClass,
+						initialArguments);
+			} catch (NoSuchMethodException e1) {
+				e1.printStackTrace();
+			}
+			this.cons = con;
+			this.buffer = getNewBuffer(initialCapacity);
+			this.gap = 200000000; // 200ms
+			expandable = true;
+			rwlock = new ReentrantReadWriteLock();
 		}
-		return newArgs;
-	}
 
-	private Buffer getNewBuffer(List<?> args) {
-		Buffer buffer;
-		try {
-			buffer = cons.newInstance(args.toArray());
-			return buffer;
-		} catch (InstantiationException | IllegalAccessException
-				| IllegalArgumentException | InvocationTargetException e) {
-			e.printStackTrace();
-			return null;
+		private List<?> getArguments(int newCapacity) {
+			List<Object> newArgs = new ArrayList<>(initialArguments.size());
+			for (int i = 0; i < initialArguments.size(); i++) {
+				if (i == capacityPos)
+					newArgs.add(newCapacity);
+				else
+					newArgs.add(initialArguments.get(i));
+			}
+			return newArgs;
 		}
-	}
 
-	@Override
-	public Object read() {
-		return buffer.read();
-	}
+		private Buffer getNewBuffer(int newCapacity) {
+			Buffer buffer;
+			try {
+				buffer = cons.newInstance(getArguments(newCapacity).toArray());
+				return buffer;
+			} catch (InstantiationException | IllegalAccessException
+					| IllegalArgumentException | InvocationTargetException e) {
+				e.printStackTrace();
+				return null;
+			}
+		}
 
-	@Override
-	public int read(Object[] data, int offset, int length) {
-		return buffer.read(data, offset, length);
-	}
+		@Override
+		public Object read() {
+			rwlock.readLock().lock();
+			Object o = buffer.read();
+			rwlock.readLock().unlock();
+			return o;
+		}
 
-	@Override
-	public boolean readAll(Object[] data) {
-		return buffer.readAll(data);
-	}
+		@Override
+		public int read(Object[] data, int offset, int length) {
+			rwlock.readLock().lock(); // TODO: getting lock for each single
+										// read. Severely affect performance.
+			int ret = buffer.read(data, offset, length);
+			rwlock.readLock().unlock();
+			return ret;
+		}
 
-	@Override
-	public boolean readAll(Object[] data, int offset) {
-		return buffer.readAll(data, offset);
-	}
+		@Override
+		public boolean readAll(Object[] data) {
+			rwlock.readLock().lock();
+			boolean ret = buffer.readAll(data);
+			rwlock.readLock().unlock();
+			return ret;
+		}
 
-	@Override
-	public boolean write(Object t) {
-		return buffer.write(t);
-	}
+		@Override
+		public boolean readAll(Object[] data, int offset) {
+			rwlock.readLock().lock();
+			boolean ret = buffer.readAll(data, offset);
+			rwlock.readLock().unlock();
+			return ret;
+		}
 
-	@Override
-	public int write(Object[] data, int offset, int length) {
-		return buffer.write(data, offset, length);
-	}
+		@Override
+		public boolean write(Object t) {
+			boolean ret = buffer.write(t);
+			if (ret)
+				lastWrittenTime = System.nanoTime();
+			else
+				writeFailed();
+			return ret;
+		}
 
-	@Override
-	public int size() {
-		return buffer.size();
-	}
+		@Override
+		public int write(Object[] data, int offset, int length) {
+			int written = buffer.write(data, offset, length);
+			if (written == 0)
+				writeFailed();
+			else
+				lastWrittenTime = System.nanoTime();
+			return written;
+		}
 
-	@Override
-	public int capacity() {
-		return buffer.capacity();
+		@Override
+		public int size() {
+			return buffer.size();
+		}
+
+		@Override
+		public int capacity() {
+			return buffer.capacity();
+		}
+
+		private void writeFailed() {
+			if (areAllFull() || !expandable)
+				return;
+
+			if (System.nanoTime() - lastWrittenTime > gap && expandable) {
+				doubleBuffer();
+			}
+		}
+
+		private boolean areAllFull() {
+			for (Buffer b : buffers) {
+				if (b.size() != b.capacity())
+					return false;
+			}
+			return true;
+		}
+
+		private void doubleBuffer() {
+			int newCapacity = 2 * buffer.capacity();
+			if (newCapacity > 1024 * initialCapacity) {
+				expandable = false;
+				return;
+			}
+
+			Buffer newBuf = getNewBuffer(newCapacity);
+			rwlock.writeLock().lock();
+			// TODO: copying is done one by one. Any block level copying?
+			for (int i = 0; i < buffer.size(); i++) {
+				newBuf.write(buffer.read());
+			}
+			if (buffer.size() != 0)
+				throw new IllegalStateException(
+						"Buffter is not empty after copying all data");
+			this.buffer = newBuf;
+			rwlock.writeLock().unlock();
+		}
 	}
 }

From 4f27a7959496feabe043d0b420f2483ef20558b2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 10 Mar 2014 23:01:18 +0800
Subject: [PATCH 326/881] Bug fix

Having i<buf.size() as for loop termination condition is wrong.
---
 .../distributed/node/DynamicBufferManager.java | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
index a0b13a74..d10d9419 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
@@ -20,12 +20,12 @@
  * @since Mar 10, 2014
  * 
  */
-public class DynamicBufferManager {
+public final class DynamicBufferManager {
 
 	/**
 	 * keep track of all buffers created for a particular blob.
 	 */
-	List<Buffer> buffers;
+	private List<Buffer> buffers;
 
 	public DynamicBufferManager() {
 		buffers = new ArrayList<>();
@@ -243,16 +243,24 @@ private void doubleBuffer() {
 				expandable = false;
 				return;
 			}
-
+			System.out
+					.println(String
+							.format("Doubling the buffer: newCapacity - %d, initialCapacity - %d",
+									newCapacity, initialCapacity));
 			Buffer newBuf = getNewBuffer(newCapacity);
 			rwlock.writeLock().lock();
+			final int size = buffer.size();
+			System.out.println("Initial buffer size = " + size);
 			// TODO: copying is done one by one. Any block level copying?
-			for (int i = 0; i < buffer.size(); i++) {
+			for (int i = 0; i < size; i++) {
 				newBuf.write(buffer.read());
 			}
-			if (buffer.size() != 0)
+
+			System.out.println("buffer size after copying = " + buffer.size());
+			if (buffer.size() != 0) {
 				throw new IllegalStateException(
 						"Buffter is not empty after copying all data");
+			}
 			this.buffer = newBuf;
 			rwlock.writeLock().unlock();
 		}

From 12b87d7f20c3ec86cd2ec1816554e9c9d0076292 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Mar 2014 00:54:31 +0800
Subject: [PATCH 327/881] Optimization - reduced the call of nanoTime()

Calls System.nanoTime() only on essential situations. In past every
successful write called this method.
---
 .../node/DynamicBufferManager.java            | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
index d10d9419..db1e586c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
@@ -132,6 +132,7 @@ public DynamicBuffer(Class<? extends Buffer> bufferClass,
 			this.gap = 200000000; // 200ms
 			expandable = true;
 			rwlock = new ReentrantReadWriteLock();
+			lastWrittenTime = 0;
 		}
 
 		private List<?> getArguments(int newCapacity) {
@@ -193,10 +194,10 @@ public boolean readAll(Object[] data, int offset) {
 		@Override
 		public boolean write(Object t) {
 			boolean ret = buffer.write(t);
-			if (ret)
-				lastWrittenTime = System.nanoTime();
-			else
+			if (!ret)
 				writeFailed();
+			else if (lastWrittenTime != 0)
+				lastWrittenTime = 0;
 			return ret;
 		}
 
@@ -205,8 +206,8 @@ public int write(Object[] data, int offset, int length) {
 			int written = buffer.write(data, offset, length);
 			if (written == 0)
 				writeFailed();
-			else
-				lastWrittenTime = System.nanoTime();
+			else if (lastWrittenTime != 0)
+				lastWrittenTime = 0;
 			return written;
 		}
 
@@ -224,6 +225,11 @@ private void writeFailed() {
 			if (areAllFull() || !expandable)
 				return;
 
+			if (lastWrittenTime == 0) {
+				lastWrittenTime = System.nanoTime();
+				return;
+			}
+
 			if (System.nanoTime() - lastWrittenTime > gap && expandable) {
 				doubleBuffer();
 			}
@@ -250,18 +256,19 @@ private void doubleBuffer() {
 			Buffer newBuf = getNewBuffer(newCapacity);
 			rwlock.writeLock().lock();
 			final int size = buffer.size();
-			System.out.println("Initial buffer size = " + size);
 			// TODO: copying is done one by one. Any block level copying?
 			for (int i = 0; i < size; i++) {
 				newBuf.write(buffer.read());
 			}
 
-			System.out.println("buffer size after copying = " + buffer.size());
+			// System.out.println("buffer size after copying = " +
+			// buffer.size());
 			if (buffer.size() != 0) {
 				throw new IllegalStateException(
 						"Buffter is not empty after copying all data");
 			}
 			this.buffer = newBuf;
+			lastWrittenTime = 0;
 			rwlock.writeLock().unlock();
 		}
 	}

From 0653e12ff8356adb71fbb7b6f0a6443c08f39d84 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Mar 2014 11:46:57 +0800
Subject: [PATCH 328/881] Gives name for each DynamicBuffer.

This is for documentation/debugging purpose.
---
 .../node/DynamicBufferManager.java            | 48 +++++++++++--------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
index db1e586c..4be5f608 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
@@ -12,8 +12,10 @@
 import edu.mit.streamjit.util.ReflectionUtils;
 
 /**
- * <b> Use a {@link DynamicBufferManager} per blob ( i.e., one to one mapping
- * between a blob and a DynamicBufferManager). Do not use same instance of
+ * Provides dynamic buffer backed by a buffer implementation which is passes an
+ * argument to {@link #getBuffer(Class, List, int, int)} method. <b> Use a
+ * {@link DynamicBufferManager} per blob ( i.e., one to one mapping between a
+ * blob and a DynamicBufferManager). Do not use same instance of
  * {@link DynamicBufferManager} to create buffers for multiple blobs. </b>
  * 
  * @author sumanan
@@ -23,7 +25,9 @@
 public final class DynamicBufferManager {
 
 	/**
-	 * keep track of all buffers created for a particular blob.
+	 * keeps track of all buffers created for a particular blob. We need to
+	 * track this list to determine whether there is an actual deadlock or this
+	 * blob is faster than all down blobs.
 	 */
 	private List<Buffer> buffers;
 
@@ -32,8 +36,15 @@ public DynamicBufferManager() {
 	}
 
 	/**
-	 * Make and return a dynamic buffer backed by an instance of bufferClass.
+	 * Makes and returns a dynamic buffer backed by an instance of bufferClass.
+	 * Passed bufferClass ( a concrete implementation of {@link Buffer}) must
+	 * have an constructor which takes the capacity of the new buffer as an
+	 * argument.
 	 * 
+	 * @param Name
+	 *            : Name for this buffer. Just for documentation purpose.
+	 *            Token.toString() may be passed where the token is a token of
+	 *            the input/output edge of a blob.
 	 * @param bufferClass
 	 *            : Any concrete implementation of {@link Buffer}.
 	 * @param initialArguments
@@ -42,9 +53,9 @@ public DynamicBufferManager() {
 	 *            : the position of size parameter in the bufferClass.
 	 * @return : A dynamic buffer backed by an instance of bufferClass.
 	 */
-	public Buffer getBuffer(Class<? extends Buffer> bufferClass,
+	public Buffer getBuffer(String name, Class<? extends Buffer> bufferClass,
 			List<?> initialArguments, int initialCapacity, int capacityPos) {
-		Buffer buf = new DynamicBuffer(bufferClass, initialArguments,
+		Buffer buf = new DynamicBuffer(name, bufferClass, initialArguments,
 				initialCapacity, capacityPos);
 		buffers.add(buf);
 		return buf;
@@ -57,9 +68,9 @@ public Buffer getBuffer(Class<? extends Buffer> bufferClass,
 	 * {@link Buffer}.
 	 * 
 	 * <p>
-	 * Determining whether buffer is full due to deadlock situation or the
+	 * Determining whether buffer fullness is due to deadlock situation or the
 	 * current blob is executing on a faster node than the down stream blob is
-	 * little tricky. Here we assume blobs are running on nearly equivalent.
+	 * little tricky.
 	 * </p>
 	 * 
 	 * <p>
@@ -71,7 +82,8 @@ public Buffer getBuffer(Class<? extends Buffer> bufferClass,
 	 * <p>
 	 * TODO: Possible performance bug during read() due to volatile buffer
 	 * variable and the need for acquire readLock for every single reading. Any
-	 * way to improve this???
+	 * way to improve this???. splitjoin1 show 30-40% performance overhead when
+	 * uses {@link DynamicBuffer}.
 	 * 
 	 * @author sumanan
 	 * @since Mar 10, 2014
@@ -79,6 +91,8 @@ public Buffer getBuffer(Class<? extends Buffer> bufferClass,
 	 */
 	private class DynamicBuffer implements Buffer {
 
+		private final String name;
+
 		/**
 		 * Minimum time gap between the last successful write and the current
 		 * time in order to consider the option of doubling the buffer
@@ -99,11 +113,10 @@ private class DynamicBuffer implements Buffer {
 
 		/**
 		 * Read lock should be acquired at every single read where as write lock
-		 * will be acquired only when switching the buffer from old to new.
+		 * only when switching the buffer from old to new.
 		 */
 		private ReadWriteLock rwlock;
 
-		private final Class<? extends Buffer> bufferClass;
 		private final List<?> initialArguments;
 		private final int initialCapacity;
 		private final int capacityPos;
@@ -114,9 +127,9 @@ private class DynamicBuffer implements Buffer {
 		 */
 		private volatile Buffer buffer;
 
-		public DynamicBuffer(Class<? extends Buffer> bufferClass,
+		public DynamicBuffer(String name, Class<? extends Buffer> bufferClass,
 				List<?> initialArguments, int initialCapacity, int capacityPos) {
-			this.bufferClass = bufferClass;
+			this.name = name;
 			this.initialArguments = initialArguments;
 			this.initialCapacity = initialCapacity;
 			this.capacityPos = capacityPos;
@@ -168,8 +181,7 @@ public Object read() {
 
 		@Override
 		public int read(Object[] data, int offset, int length) {
-			rwlock.readLock().lock(); // TODO: getting lock for each single
-										// read. Severely affect performance.
+			rwlock.readLock().lock();
 			int ret = buffer.read(data, offset, length);
 			rwlock.readLock().unlock();
 			return ret;
@@ -251,8 +263,8 @@ private void doubleBuffer() {
 			}
 			System.out
 					.println(String
-							.format("Doubling the buffer: newCapacity - %d, initialCapacity - %d",
-									newCapacity, initialCapacity));
+							.format("%s : Doubling the buffer: initialCapacity - %d, newCapacity - %d",
+									name, initialCapacity, newCapacity));
 			Buffer newBuf = getNewBuffer(newCapacity);
 			rwlock.writeLock().lock();
 			final int size = buffer.size();
@@ -261,8 +273,6 @@ private void doubleBuffer() {
 				newBuf.write(buffer.read());
 			}
 
-			// System.out.println("buffer size after copying = " +
-			// buffer.size());
 			if (buffer.size() != 0) {
 				throw new IllegalStateException(
 						"Buffter is not empty after copying all data");

From e8a7475de90b8b51d565c667edc21efe609c9c03 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Mar 2014 12:08:37 +0800
Subject: [PATCH 329/881] Binds a DynamicBufferManager to a blob.

An instance of DynamicBufferManger can generate dynamic buffer of a single
blob only. Same instance cannot be used to generate dynamic buffers for several
blobs.
---
 .../node/DynamicBufferManager.java            | 40 +++++++++++++++----
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
index 4be5f608..7c308403 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/DynamicBufferManager.java
@@ -4,19 +4,25 @@
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.util.ConstructorSupplier;
 import edu.mit.streamjit.util.ReflectionUtils;
 
 /**
- * Provides dynamic buffer backed by a buffer implementation which is passes an
- * argument to {@link #getBuffer(Class, List, int, int)} method. <b> Use a
+ * Provides dynamic buffer backed by a buffer implementation which is passed as
+ * an argument to {@link #getBuffer(Class, List, int, int)} method. <b> Use a
  * {@link DynamicBufferManager} per blob ( i.e., one to one mapping between a
  * blob and a DynamicBufferManager). Do not use same instance of
- * {@link DynamicBufferManager} to create buffers for multiple blobs. </b>
+ * {@link DynamicBufferManager} to create buffers for multiple blobs or multiple
+ * {@link DynamicBufferManager}s to make buffer for a single blob. Doing this
+ * will affect the deadlock detection algorithm and ultimately results either
+ * infinite buffer growth or unsolvable deadlock. </b>
  * 
  * @author sumanan
  * @since Mar 10, 2014
@@ -24,6 +30,16 @@
  */
 public final class DynamicBufferManager {
 
+	/**
+	 * inputs of the blob in which this object is bound to.
+	 */
+	private final Set<Token> inputs;
+
+	/**
+	 * outputs of the blob in which this object is bound to.
+	 */
+	private final Set<Token> outputs;
+
 	/**
 	 * keeps track of all buffers created for a particular blob. We need to
 	 * track this list to determine whether there is an actual deadlock or this
@@ -31,7 +47,9 @@ public final class DynamicBufferManager {
 	 */
 	private List<Buffer> buffers;
 
-	public DynamicBufferManager() {
+	public DynamicBufferManager(Blob blob) {
+		inputs = blob.getInputs();
+		outputs = blob.getOutputs();
 		buffers = new ArrayList<>();
 	}
 
@@ -53,14 +71,20 @@ public DynamicBufferManager() {
 	 *            : the position of size parameter in the bufferClass.
 	 * @return : A dynamic buffer backed by an instance of bufferClass.
 	 */
-	public Buffer getBuffer(String name, Class<? extends Buffer> bufferClass,
+	public Buffer getBuffer(Token t, Class<? extends Buffer> bufferClass,
 			List<?> initialArguments, int initialCapacity, int capacityPos) {
-		Buffer buf = new DynamicBuffer(name, bufferClass, initialArguments,
-				initialCapacity, capacityPos);
+
+		if (!(outputs.contains(t) || inputs.contains(t)))
+			throw new IllegalAccessError(
+					String.format(
+							"%s is not related to the blob in which this dynamic buffer manager is bound to",
+							t.toString()));;
+
+		Buffer buf = new DynamicBuffer(t.toString(), bufferClass,
+				initialArguments, initialCapacity, capacityPos);
 		buffers.add(buf);
 		return buf;
 	}
-
 	/**
 	 * Dynamically increases supplied buffer capacity in order to avoid dead
 	 * locks. Actually creates a new instance of the supplied buffer and copy

From af65cd0865151b635d939a44c026e25808c1fb66 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 16 Sep 2014 01:44:36 +0800
Subject: [PATCH 330/881] Private method clean(Buffer b, Token t) added.

Just refactored and moved the repeating code into a new private method.
---
 .../distributed/node/BlobsManagerImpl.java    | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index ac254c83..7cea5597 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -789,25 +789,12 @@ public void run() {
 						areAllDrained = false;
 						for (Token t : be.blob.getOutputs()) {
 							Buffer b = be.bufferMap.get(t);
-							int size = b.size();
-							if (size == 0)
-								continue;
-							System.out.println(String.format(
-									"Buffer %s has %d data. Going to clean it",
-									t.toString(), size));
-							Object[] obArray = new Object[size];
-							b.readAll(obArray);
+							clean(b, t);
 						}
+
 						for (Token t : be.blob.getInputs()) {
 							Buffer b = be.bufferMap.get(t);
-							int size = b.size();
-							if (size == 0)
-								continue;
-							System.out.println(String.format(
-									"Buffer %s has %d data. Going to clean it",
-									t.toString(), size));
-							Object[] obArray = new Object[size];
-							b.readAll(obArray);
+							clean(b, t);
 						}
 					}
 				}
@@ -817,6 +804,17 @@ public void run() {
 			}
 		}
 
+		private void clean(Buffer b, Token t) {
+			int size = b.size();
+			if (size == 0)
+				return;
+			System.out.println(String.format(
+					"Buffer %s has %d data. Going to clean it", t.toString(),
+					size));
+			Object[] obArray = new Object[size];
+			b.readAll(obArray);
+		}
+
 		public void stopit() {
 			this.run.set(false);
 			this.interrupt();

From 05cb1b1fbf5e7d40ec99ec2ffa283d9a5a17144f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 16 Sep 2014 13:58:07 +0800
Subject: [PATCH 331/881] DrainDeadLockHandler refactored.

Code in the run() method is moved to a new private method cleanAllBuffers().
---
 .../distributed/node/BlobsManagerImpl.java    | 43 ++++++++++++-------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 7cea5597..abf0657c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -782,26 +782,37 @@ public void run() {
 			boolean areAllDrained = false;
 
 			while (run.get()) {
-				areAllDrained = true;
-				for (BlobExecuter be : blobExecuters.values()) {
-					if (be.drainState == 1 || be.drainState == 2) {
-						// System.out.println(be.blobID + " is not drained");
-						areAllDrained = false;
-						for (Token t : be.blob.getOutputs()) {
-							Buffer b = be.bufferMap.get(t);
-							clean(b, t);
-						}
+				areAllDrained = cleanAllBuffers();
+				if (areAllDrained)
+					break;
+			}
+		}
 
-						for (Token t : be.blob.getInputs()) {
-							Buffer b = be.bufferMap.get(t);
-							clean(b, t);
-						}
+		/**
+		 * Go through all blocked blobs and clean all input and output buffers.
+		 * This method is useful when we don't care about the drain data.
+		 * 
+		 * @return true iff there is no blocked blobs, i.e., all blobs have
+		 *         completed the draining.
+		 */
+		private boolean cleanAllBuffers() {
+			boolean areAllDrained = true;
+			for (BlobExecuter be : blobExecuters.values()) {
+				if (be.drainState == 1 || be.drainState == 2) {
+					// System.out.println(be.blobID + " is not drained");
+					areAllDrained = false;
+					for (Token t : be.blob.getOutputs()) {
+						Buffer b = be.bufferMap.get(t);
+						clean(b, t);
 					}
-				}
 
-				if (areAllDrained)
-					break;
+					for (Token t : be.blob.getInputs()) {
+						Buffer b = be.bufferMap.get(t);
+						clean(b, t);
+					}
+				}
 			}
+			return areAllDrained;
 		}
 
 		private void clean(Buffer b, Token t) {

From d39741af84d21f0b602e2a52b08d90f0a09a8f1c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 17 Sep 2014 10:08:11 +0800
Subject: [PATCH 332/881] Code cleaning : Obsolete commented codes removed.

---
 .../distributed/node/BlobsManagerImpl.java    | 29 +++----------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index abf0657c..9737e068 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -121,21 +121,8 @@ public CTRLRDrainProcessor getDrainProcessor() {
 	}
 
 	public void reqDrainedData(Set<Token> blobSet) {
-		// ImmutableMap.Builder<Token, DrainData> builder = new
-		// ImmutableMap.Builder<>();
-		// for (BlobExecuter be : blobExecuters) {
-		// if (be.isDrained) {
-		// builder.put(be.blobID, be.blob.getDrainData());
-		// }
-		// }
-		//
-		// try {
-		// streamNode.controllerConnection
-		// .writeObject(new SNDrainElement.DrainedData(builder.build()));
-		// } catch (IOException e) {
-		// // TODO Auto-generated catch block
-		// e.printStackTrace();
-		// }
+		throw new UnsupportedOperationException(
+				"Method reqDrainedData not implemented");
 	}
 
 	/**
@@ -383,18 +370,10 @@ private void doDrain(boolean reqDrainData) {
 				stopType = 3;
 
 			inChnlManager.stop(stopType);
-			// inChnlManager.waitToStop();
-
 			// TODO: [2014-03-14] I commented following lines to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
-			// for (Thread t : inputChannelThreads) {
-			// try {
-			// t.join();
-			// } catch (InterruptedException e) {
-			// e.printStackTrace();
-			// }
-			// }
-			// System.out.println("Blob " + blobID + "Input thread's joined");
+			// inChnlManager.waitToStop();
+
 			if (this.blob != null) {
 				DrainCallback dcb = new DrainCallback(this);
 				drainState = 2;

From 0f26da5640d6336fde6c1008c7fe33e241613740 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 17 Sep 2014 10:17:19 +0800
Subject: [PATCH 333/881] getDrainData() calls  inChnlManager.waitToStop()

Before copying unprocessed data from BoundaryChannels, all channel threads
must be stopped. As inChnlManager.waitToStop() is commented inside doDrain()
due to deadlock 5, We need to call it at getDrainData() to ensure input channel
threads are stopped.
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 9737e068..c4f9be5a 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -479,9 +479,14 @@ private DrainedData getDrainData() {
 
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
-
 			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
 					.inputChannelsMap();
+
+			// In a proper system the following line should be called inside
+			// doDrain(), just after inChnlManager.stop(). Read the comment
+			// in doDrain().
+			inChnlManager.waitToStop();
+
 			for (Token t : blob.getInputs()) {
 				if (inputChannels.containsKey(t)) {
 					BoundaryChannel chanl = inputChannels.get(t);

From 57dbfc1d5a72919305427c15ff232414e60ea517 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 17 Sep 2014 12:25:46 +0800
Subject: [PATCH 334/881] doDrain() calls inChnlManager.waitToStop()

if drain data is required
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index c4f9be5a..284759ab 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -370,9 +370,11 @@ private void doDrain(boolean reqDrainData) {
 				stopType = 3;
 
 			inChnlManager.stop(stopType);
-			// TODO: [2014-03-14] I commented following lines to avoid one dead
+			// TODO: [2014-03-14] I commented following line to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
-			// inChnlManager.waitToStop();
+			// [2014-09-17] Lets waitToStop() if drain data is required.
+			if (stopType != 3)
+				inChnlManager.waitToStop();
 
 			if (this.blob != null) {
 				DrainCallback dcb = new DrainCallback(this);

From a9ebd058a6577c7fe90e8688e7f7517b7ea0e642 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 18 Sep 2014 06:58:53 +0800
Subject: [PATCH 335/881] PerformanceLogger prints exit message.

PerformanceLogger prints exit message and write the output count into the file.
---
 .../impl/distributed/TailChannel.java         | 30 ++++++++++++++++---
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index ed70f44b..11d791a2 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -27,7 +27,7 @@ public class TailChannel extends BlockingInputChannel {
 
 	private volatile CountDownLatch skipLatch;
 
-	private performanceLogger pLogger = null;
+	private PerformanceLogger pLogger = null;
 
 	private boolean skipLatchUp;
 
@@ -58,7 +58,7 @@ public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 		skipLatch = new CountDownLatch(1);
 		this.skipLatchUp = true;
 		if (GlobalConstants.tune == 0) {
-			pLogger = new performanceLogger();
+			pLogger = new PerformanceLogger();
 			pLogger.start();
 		}
 	}
@@ -131,11 +131,11 @@ public void releaseAll() {
 		count = 0;
 	}
 
-	private class performanceLogger extends Thread {
+	private class PerformanceLogger extends Thread {
 
 		private AtomicBoolean stopFlag;
 
-		private performanceLogger() {
+		private PerformanceLogger() {
 			stopFlag = new AtomicBoolean(false);
 		}
 
@@ -148,6 +148,9 @@ public void run() {
 				e1.printStackTrace();
 				return;
 			}
+
+			writeInitialInfo(writer);
+
 			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Long time = getFixedOutputTime();
@@ -169,6 +172,25 @@ public void run() {
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
+
+			System.out.println("PerformanceLogger exits. App will run till "
+					+ "inputdata exhausted.");
+		}
+
+		private void writeInitialInfo(FileWriter writer) {
+			System.out.println(String.format(
+					"PerformanceLogger starts to log the time to"
+							+ " produce %d number of outputs",
+					GlobalConstants.outputCount));
+
+			try {
+				writer.write(String.format("GlobalConstants.outputCount = %d",
+						GlobalConstants.outputCount));
+				writer.write('\n');
+				writer.flush();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 		}
 
 		public void stopLogging() {

From 48b69daab429c61295ddaf7c77bea8b1299ce233 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 18 Sep 2014 08:04:16 +0800
Subject: [PATCH 336/881] evaluate() calls terminate() to close the execution

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 79895d24..4de9ef11 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -74,8 +74,7 @@ private void tune() {
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
-				if (cfgJson == null)
-				{
+				if (cfgJson == null) {
 					System.err.println("OpenTuner closed unexpectly.");
 					break;
 				}
@@ -128,11 +127,7 @@ private void evaluate() {
 				app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
-		if (manager.isRunning()) {
-			drainer.startDraining(1);
-		} else {
-			manager.stop();
-		}
+		terminate();
 	}
 
 	/**

From 9e6cdc4afc7deba8c465de03be56ebd1bee0ec18 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 04:52:23 +0800
Subject: [PATCH 337/881] Logs the drainData statistic in evaluate() mode.

Already, we log drainData statistic in tune() mode. Now doing the same for
evaluate() mode.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4de9ef11..e12b9625 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -127,6 +127,12 @@ private void evaluate() {
 				app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
+		try {
+			drainer.dumpDraindataStatistics();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
 		terminate();
 	}
 

From 4cc923b80380ea808353b4ee022cd0753ab8688b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 05:37:26 +0800
Subject: [PATCH 338/881] Bug Fix : Avoids closing connections multiple time

Sometimes two or more threads call controller.closeAll(). This causes exception
as later callers try to write EXIT message on the Connections which are already
closed by the first caller.

E.g., HeadThread calls StreamJitAppManager.stop() when there is no more input,
following that OnlineTuner thread also calls to stop().
---
 .../distributed/runtimer/BlockingCommunicationManager.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
index e24fe5db..c9a08847 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
@@ -208,7 +208,8 @@ public void run() {
 		public void close() {
 			try {
 				SNAgent.stopRequest();
-				connection.writeObject(Request.EXIT);
+				if (connection.isStillConnected())
+					connection.writeObject(Request.EXIT);
 				connection.closeConnection();
 			} catch (IOException e) {
 				// TODO Auto-generated catch block

From 17f618f84560706f54c8d8d1b0c57c754229aab1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 06:38:06 +0800
Subject: [PATCH 339/881] Bug fix : reconfigure() stops if app has stopped.

App may have already stopped by HeadChannel when tune() calls reconfigure().
So reconfigure() checks whether app has stopped or not before doing the actual
reconfiguration steps.
---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e12b9625..7f1d347a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -144,6 +144,10 @@ private void evaluate() {
 	 */
 	private Pair<Boolean, Long> reconfigure(Configuration config) {
 		long time;
+
+		if (manager.getStatus() == AppStatus.STOPPED)
+			return new Pair<Boolean, Long>(false, 0l);
+
 		try {
 			if (!cfgManager.newConfiguration(config)) {
 				return new Pair<Boolean, Long>(true, -1l);

From 7793e731dccfecfb853632b6797ba152f374495b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 09:30:22 +0800
Subject: [PATCH 340/881] Members sorted.

Just used Eclipse's feature to sort members.
---
 .../DistributedStreamCompiler.java            | 80 ++++++++++---------
 1 file changed, 41 insertions(+), 39 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index a6afb7df..7ae53bb3 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -75,6 +75,13 @@ public class DistributedStreamCompiler implements StreamCompiler {
 	 */
 	private int noOfnodes;
 
+	/**
+	 * Run the whole application on the controller node.
+	 */
+	public DistributedStreamCompiler() {
+		this(1);
+	}
+
 	/**
 	 * @param noOfnodes
 	 *            : Total number of nodes the stream application intended to run
@@ -87,13 +94,6 @@ public DistributedStreamCompiler(int noOfnodes) {
 		this.noOfnodes = noOfnodes;
 	}
 
-	/**
-	 * Run the whole application on the controller node.
-	 */
-	public DistributedStreamCompiler() {
-		this(1);
-	}
-
 	/**
 	 * Run the application with the passed configureation.
 	 */
@@ -233,18 +233,25 @@ public void drain() {
 		return cs;
 	}
 
-	private Configuration readConfiguration(String simpeName) {
-		String name = String.format("%s.cfg", simpeName);
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			reader.close();
-			return Configuration.fromJson(json);
-		} catch (Exception ex) {
-			System.err.println(String.format(
-					"File reader error. No %s configuration file.", name));
+	/**
+	 * TODO: Need to check for other default subtypes of {@link OneToOneElement}
+	 * s. Now only checks for first generation children.
+	 * 
+	 * @param stream
+	 * @throws StreamCompilationFailedException
+	 *             if stream is default subtype of OneToOneElement
+	 */
+	private <I, O> void checkforDefaultOneToOneElement(
+			OneToOneElement<I, O> stream) {
+
+		if (stream.getClass() == Pipeline.class
+				|| stream.getClass() == Splitjoin.class
+				|| stream.getClass() == Filter.class) {
+			throw new StreamCompilationFailedException(
+					"Default subtypes of OneToOneElement are not accepted for"
+							+ " compilation by this compiler. OneToOneElement"
+							+ " that passed should be unique");
 		}
-		return null;
 	}
 
 	private <I, O> Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
@@ -274,23 +281,18 @@ private Configuration readConfiguration(String simpeName) {
 		return partitionsMachineMap;
 	}
 
-	/**
-	 * TODO: Need to check for other default subtypes of {@link OneToOneElement}
-	 * s. Now only checks for first generation children.
-	 * 
-	 * @param stream
-	 * @throws StreamCompilationFailedException
-	 *             if stream is default subtype of OneToOneElement
-	 */
-	private <I, O> void checkforDefaultOneToOneElement(
-			OneToOneElement<I, O> stream) {
-
-		if (stream.getClass() == Pipeline.class
-				|| stream.getClass() == Splitjoin.class
-				|| stream.getClass() == Filter.class) {
-			throw new StreamCompilationFailedException(
-					"Default subtypes of OneToOneElement are not accepted for compilation by this compiler. OneToOneElement that passed should be unique");
+	private Configuration readConfiguration(String simpeName) {
+		String name = String.format("%s.cfg", simpeName);
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
 		}
+		return null;
 	}
 
 	private static class DistributedCompiledStream implements CompiledStream {
@@ -301,11 +303,6 @@ public DistributedCompiledStream(AbstractDrainer drainer) {
 			this.drainer = drainer;
 		}
 
-		@Override
-		public boolean isDrained() {
-			return drainer.isDrained();
-		}
-
 		@Override
 		public void awaitDrained() throws InterruptedException {
 			drainer.awaitDrained();
@@ -317,6 +314,11 @@ public void awaitDrained(long timeout, TimeUnit unit)
 				throws InterruptedException, TimeoutException {
 			drainer.awaitDrained(timeout, unit);
 		}
+
+		@Override
+		public boolean isDrained() {
+			return drainer.isDrained();
+		}
 	}
 
 	private class IllegalConfigurationException extends RuntimeException {

From 446f9b86d6c185e647fa11538920d01dfa75141f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 12:37:26 +0800
Subject: [PATCH 341/881] DSC.compile() refactored. visit() added.

---
 .../DistributedStreamCompiler.java            | 40 +++++++++++--------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 7ae53bb3..3593f1e7 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -47,6 +47,7 @@
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
+import edu.mit.streamjit.util.Pair;
 
 /**
  * 
@@ -106,16 +107,7 @@ public DistributedStreamCompiler(int noOfnodes, Configuration cfg) {
 
 	public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			Input<I> input, Output<O> output) {
-
-		checkforDefaultOneToOneElement(stream);
-
-		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
-		stream.visit(primitiveConnector);
-		Worker<I, ?> source = (Worker<I, ?>) primitiveConnector.getSource();
-		Worker<?, O> sink = (Worker<?, O>) primitiveConnector.getSink();
-
-		VerifyStreamGraph verifier = new VerifyStreamGraph();
-		stream.visit(verifier);
+		Pair<Worker<I, ?>, Worker<?, O>> srcSink = visit(stream);
 
 		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
 
@@ -133,14 +125,15 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		Controller controller = new Controller();
 		controller.connect(conTypeCount);
 
-		StreamJitApp app = new StreamJitApp(stream, source, sink);
+		StreamJitApp app = new StreamJitApp(stream, srcSink.first,
+				srcSink.second);
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));
 		this.cfg = bf.getDefaultConfiguration(Workers
-				.getAllWorkersInGraph(source));
+				.getAllWorkersInGraph(srcSink.first));
 
 		if (GlobalConstants.tune == 0) {
 			Configuration cfg1 = readConfiguration(stream.getClass()
@@ -168,7 +161,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 				machineIds[i] = i + 1;
 			}
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-					machineIds, stream, source, sink);
+					machineIds, stream, srcSink.first, srcSink.second);
 			app.newPartitionMap(partitionsMachineMap);
 		} else
 			cfgManager.newConfiguration(cfg);
@@ -176,7 +169,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
 		// context.
 		List<MessageConstraint> constraints = MessageConstraint
-				.findConstraints(source);
+				.findConstraints(srcSink.first);
 		Set<Portal<?>> portals = new HashSet<>();
 		for (MessageConstraint mc : constraints)
 			portals.add(mc.getPortal());
@@ -216,8 +209,10 @@ public void drain() {
 		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
 				.<Token, Buffer> builder();
 
-		bufferMapBuilder.put(Token.createOverallInputToken(source), head);
-		bufferMapBuilder.put(Token.createOverallOutputToken(sink), tail);
+		bufferMapBuilder
+				.put(Token.createOverallInputToken(srcSink.first), head);
+		bufferMapBuilder.put(Token.createOverallOutputToken(srcSink.second),
+				tail);
 
 		app.bufferMap = bufferMapBuilder.build();
 		app.constraints = constraints;
@@ -295,6 +290,19 @@ private Configuration readConfiguration(String simpeName) {
 		return null;
 	}
 
+	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
+			OneToOneElement<I, O> stream) {
+		checkforDefaultOneToOneElement(stream);
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<I, ?> source = (Worker<I, ?>) primitiveConnector.getSource();
+		Worker<?, O> sink = (Worker<?, O>) primitiveConnector.getSink();
+
+		VerifyStreamGraph verifier = new VerifyStreamGraph();
+		stream.visit(verifier);
+		return new Pair<Worker<I, ?>, Worker<?, O>>(source, sink);
+	}
+
 	private static class DistributedCompiledStream implements CompiledStream {
 
 		AbstractDrainer drainer;

From f354064e212df4ca40a6ac20088b6b9b27e0b281 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 15:51:46 +0800
Subject: [PATCH 342/881] Constructors of DSC organized.

---
 .../DistributedStreamCompiler.java            | 44 +++++++++++--------
 1 file changed, 26 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 3593f1e7..d82cacc1 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -77,31 +77,46 @@ public class DistributedStreamCompiler implements StreamCompiler {
 	private int noOfnodes;
 
 	/**
-	 * Run the whole application on the controller node.
+	 * Run the whole application on the controller node. No distributions. See
+	 * {@link #DistributedStreamCompiler(int, Configuration)}
 	 */
 	public DistributedStreamCompiler() {
-		this(1);
+		this(1, null);
 	}
 
 	/**
-	 * @param noOfnodes
-	 *            : Total number of nodes the stream application intended to run
-	 *            - including controller node. If it is 1 then it means the
-	 *            whole stream application is supposed to run on controller.
+	 * See {@link #DistributedStreamCompiler(int, Configuration)}. As no
+	 * configuration is passed, tuner will activated to tune for better
+	 * configuration.
 	 */
 	public DistributedStreamCompiler(int noOfnodes) {
-		if (noOfnodes < 1)
-			throw new IllegalArgumentException("noOfnodes must be 1 or greater");
-		this.noOfnodes = noOfnodes;
+		this(noOfnodes, null);
 	}
 
 	/**
-	 * Run the application with the passed configureation.
+	 * Run the application with the passed configuration. Pass null if the
+	 * intention is to tune the application.
+	 * 
+	 * @param noOfnodes
+	 *            : Total number of nodes the stream application intended to run
+	 *            including the controller node. If it is 1 then it means the
+	 *            whole stream application is supposed to run on controller.
+	 * @param cfg
+	 *            Run the application with the passed {@link Configuration}. If
+	 *            it is null, tuner will be activated to tune for better
+	 *            configuration.
 	 */
 	public DistributedStreamCompiler(int noOfnodes, Configuration cfg) {
 		if (noOfnodes < 1)
 			throw new IllegalArgumentException("noOfnodes must be 1 or greater");
-		this.noOfnodes = noOfnodes;
+		if (GlobalConstants.singleNodeOnline) {
+			System.out
+					.println("Flag GlobalConstants.singleNodeOnline is enabled."
+							+ " noOfNodes passed as compiler argument has no effect");
+			this.noOfnodes = 1;
+		} else
+			this.noOfnodes = noOfnodes;
+
 		this.cfg = cfg;
 	}
 
@@ -111,13 +126,6 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
 
-		if (GlobalConstants.singleNodeOnline) {
-			System.out
-					.println("Flag GlobalConstants.singleNodeOnline is enabled."
-							+ " noOfNodes passed as compiler argument has no effect");
-			this.noOfnodes = 1;
-		}
-
 		if (this.noOfnodes == 1)
 			conTypeCount.put(CommunicationType.LOCAL, 1);
 		else

From e4b8f9a369087346eeb948f0d5b2d9c43ce17fb2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 16:00:50 +0800
Subject: [PATCH 343/881] DSC.compile() refactored.

establishController() added.
---
 .../DistributedStreamCompiler.java            | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index d82cacc1..c5d17f5b 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -123,15 +123,7 @@ public DistributedStreamCompiler(int noOfnodes, Configuration cfg) {
 	public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			Input<I> input, Output<O> output) {
 		Pair<Worker<I, ?>, Worker<?, O>> srcSink = visit(stream);
-
-		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
-
-		if (this.noOfnodes == 1)
-			conTypeCount.put(CommunicationType.LOCAL, 1);
-		else
-			conTypeCount.put(CommunicationType.TCP, this.noOfnodes - 1);
-		Controller controller = new Controller();
-		controller.connect(conTypeCount);
+		Controller controller = establishController();
 
 		StreamJitApp app = new StreamJitApp(stream, srcSink.first,
 				srcSink.second);
@@ -257,6 +249,18 @@ private <I, O> void checkforDefaultOneToOneElement(
 		}
 	}
 
+	private Controller establishController() {
+		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
+
+		if (this.noOfnodes == 1)
+			conTypeCount.put(CommunicationType.LOCAL, 1);
+		else
+			conTypeCount.put(CommunicationType.TCP, this.noOfnodes - 1);
+		Controller controller = new Controller();
+		controller.connect(conTypeCount);
+		return controller;
+	}
+
 	private <I, O> Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
 			Integer[] machineIds, OneToOneElement<I, O> stream,
 			Worker<I, ?> source, Worker<?, O> sink) {

From 85bd1718a2c5f610f59f3a86a31f2bce0f8b166e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 16:50:13 +0800
Subject: [PATCH 344/881] DSC.compile() refactored. setConfiguration() added

---
 .../DistributedStreamCompiler.java            | 79 ++++++++++---------
 1 file changed, 43 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index c5d17f5b..53f2895f 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -124,47 +124,13 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			Input<I> input, Output<O> output) {
 		Pair<Worker<I, ?>, Worker<?, O>> srcSink = visit(stream);
 		Controller controller = establishController();
-
 		StreamJitApp app = new StreamJitApp(stream, srcSink.first,
 				srcSink.second);
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
-				Math.max(noOfnodes - 1, 1));
-		this.cfg = bf.getDefaultConfiguration(Workers
-				.getAllWorkersInGraph(srcSink.first));
-
-		if (GlobalConstants.tune == 0) {
-			Configuration cfg1 = readConfiguration(stream.getClass()
-					.getSimpleName());
-			if (cfg1 == null) {
-				controller.closeAll();
-				throw new IllegalConfigurationException();
-			} else if (!this.cfg.getParametersMap().keySet()
-					.equals(cfg1.getParametersMap().keySet())) {
-				System.err
-						.println("Reading the configuration from configuration file");
-				System.err
-						.println("No matching between parameters in the read configuration and parameters in the default configuration");
-				controller.closeAll();
-				throw new IllegalConfigurationException();
-			}
-			this.cfg = cfg1;
-		}
-
-		if (cfg == null) {
-			System.err
-					.println("Configuration is null. Runs the app with horizontal partitioning.");
-			Integer[] machineIds = new Integer[this.noOfnodes - 1];
-			for (int i = 0; i < machineIds.length; i++) {
-				machineIds[i] = i + 1;
-			}
-			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-					machineIds, stream, srcSink.first, srcSink.second);
-			app.newPartitionMap(partitionsMachineMap);
-		} else
-			cfgManager.newConfiguration(cfg);
+		setConfiguration(controller, srcSink, stream, app, cfgManager,
+				conManager);
 
 		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
 		// context.
@@ -302,6 +268,47 @@ private Configuration readConfiguration(String simpeName) {
 		return null;
 	}
 
+	private <I, O> void setConfiguration(Controller controller,
+			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
+			OneToOneElement<I, O> stream, StreamJitApp app,
+			ConfigurationManager cfgManager, ConnectionManager conManager) {
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
+				Math.max(noOfnodes - 1, 1));
+		this.cfg = bf.getDefaultConfiguration(Workers
+				.getAllWorkersInGraph(srcSink.first));
+		if (GlobalConstants.tune == 0) {
+			Configuration cfg1 = readConfiguration(stream.getClass()
+					.getSimpleName());
+			if (cfg1 == null) {
+				controller.closeAll();
+				throw new IllegalConfigurationException();
+			} else if (!this.cfg.getParametersMap().keySet()
+					.equals(cfg1.getParametersMap().keySet())) {
+				System.err
+						.println("Reading the configuration from configuration file");
+				System.err
+						.println("No matching between parameters in the read "
+								+ "configuration and parameters in the default configuration");
+				controller.closeAll();
+				throw new IllegalConfigurationException();
+			}
+			this.cfg = cfg1;
+		}
+
+		if (cfg == null) {
+			System.err
+					.println("Configuration is null. Runs the app with horizontal partitioning.");
+			Integer[] machineIds = new Integer[this.noOfnodes - 1];
+			for (int i = 0; i < machineIds.length; i++) {
+				machineIds[i] = i + 1;
+			}
+			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
+					machineIds, stream, srcSink.first, srcSink.second);
+			app.newPartitionMap(partitionsMachineMap);
+		} else
+			cfgManager.newConfiguration(cfg);
+	}
+
 	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
 			OneToOneElement<I, O> stream) {
 		checkforDefaultOneToOneElement(stream);

From a659ebd9cbdfff02912e4e628bb86e8d0c9c9e6d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 17:31:28 +0800
Subject: [PATCH 345/881] DSC.compile() refactored. setBufferMap() added.

---
 .../DistributedStreamCompiler.java            | 79 +++++++++++--------
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 53f2895f..05ec3340 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -147,40 +147,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		final AbstractDrainer drainer = new DistributedDrainer(manager);
 		drainer.setBlobGraph(app.blobGraph);
 
-		// TODO: derive a algorithm to find good buffer size and use here.
-		Buffer head = InputBufferFactory.unwrap(input).createReadableBuffer(
-				10000);
-		Buffer tail = OutputBufferFactory.unwrap(output).createWritableBuffer(
-				10000);
-
-		boolean needTermination;
-
-		if (input instanceof ManualInput) {
-			needTermination = false;
-			InputBufferFactory
-					.setManualInputDelegate(
-							(ManualInput<I>) input,
-							new InputBufferFactory.AbstractManualInputDelegate<I>(
-									head) {
-								@Override
-								public void drain() {
-									drainer.startDraining(2);
-								}
-							});
-		} else {
-			needTermination = true;
-			head = new HeadBuffer(head, drainer);
-		}
-
-		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-				.<Token, Buffer> builder();
-
-		bufferMapBuilder
-				.put(Token.createOverallInputToken(srcSink.first), head);
-		bufferMapBuilder.put(Token.createOverallOutputToken(srcSink.second),
-				tail);
-
-		app.bufferMap = bufferMapBuilder.build();
+		boolean needTermination = setBufferMap(input, output, drainer, app,
+				srcSink);
 		app.constraints = constraints;
 
 		manager.reconfigure(1);
@@ -268,6 +236,49 @@ private Configuration readConfiguration(String simpeName) {
 		return null;
 	}
 
+	/**
+	 * Sets head and tail buffers.
+	 */
+	private <I, O> boolean setBufferMap(Input<I> input, Output<O> output,
+			final AbstractDrainer drainer, StreamJitApp app,
+			Pair<Worker<I, ?>, Worker<?, O>> srcSink) {
+		// TODO: derive a algorithm to find good buffer size and use here.
+		Buffer head = InputBufferFactory.unwrap(input).createReadableBuffer(
+				10000);
+		Buffer tail = OutputBufferFactory.unwrap(output).createWritableBuffer(
+				10000);
+
+		boolean needTermination;
+
+		if (input instanceof ManualInput) {
+			needTermination = false;
+			InputBufferFactory
+					.setManualInputDelegate(
+							(ManualInput<I>) input,
+							new InputBufferFactory.AbstractManualInputDelegate<I>(
+									head) {
+								@Override
+								public void drain() {
+									drainer.startDraining(2);
+								}
+							});
+		} else {
+			needTermination = true;
+			head = new HeadBuffer(head, drainer);
+		}
+
+		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+				.<Token, Buffer> builder();
+
+		bufferMapBuilder
+				.put(Token.createOverallInputToken(srcSink.first), head);
+		bufferMapBuilder.put(Token.createOverallOutputToken(srcSink.second),
+				tail);
+
+		app.bufferMap = bufferMapBuilder.build();
+		return needTermination;
+	}
+
 	private <I, O> void setConfiguration(Controller controller,
 			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
 			OneToOneElement<I, O> stream, StreamJitApp app,

From 40b0a85a8a3215717cd69a4000720e23e397b815 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Sep 2014 17:36:39 +0800
Subject: [PATCH 346/881] DSC.compile() refactored. setConstrains() added.

---
 .../DistributedStreamCompiler.java            | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 05ec3340..ed4308be 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -132,16 +132,6 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		setConfiguration(controller, srcSink, stream, app, cfgManager,
 				conManager);
 
-		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
-		// context.
-		List<MessageConstraint> constraints = MessageConstraint
-				.findConstraints(srcSink.first);
-		Set<Portal<?>> portals = new HashSet<>();
-		for (MessageConstraint mc : constraints)
-			portals.add(mc.getPortal());
-		for (Portal<?> portal : portals)
-			Portals.setConstraints(portal, constraints);
-
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
 				cfgManager, conManager);
 		final AbstractDrainer drainer = new DistributedDrainer(manager);
@@ -149,7 +139,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		boolean needTermination = setBufferMap(input, output, drainer, app,
 				srcSink);
-		app.constraints = constraints;
+		setConstrains(srcSink, app);
 
 		manager.reconfigure(1);
 		CompiledStream cs = new DistributedCompiledStream(drainer);
@@ -320,6 +310,20 @@ private <I, O> void setConfiguration(Controller controller,
 			cfgManager.newConfiguration(cfg);
 	}
 
+	private <I, O> void setConstrains(Pair<Worker<I, ?>, Worker<?, O>> srcSink,
+			StreamJitApp app) {
+		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
+		// context.
+		List<MessageConstraint> constraints = MessageConstraint
+				.findConstraints(srcSink.first);
+		Set<Portal<?>> portals = new HashSet<>();
+		for (MessageConstraint mc : constraints)
+			portals.add(mc.getPortal());
+		for (Portal<?> portal : portals)
+			Portals.setConstraints(portal, constraints);
+		app.constraints = constraints;
+	}
+
 	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
 			OneToOneElement<I, O> stream) {
 		checkforDefaultOneToOneElement(stream);

From 1d2cdb33a2d0c44d41afa180c8fef8c026377078 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Sep 2014 07:48:22 +0800
Subject: [PATCH 347/881] DSC.compile() refactored. manualPartition() added.

---
 .../DistributedStreamCompiler.java            | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index ed4308be..a2681f9c 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -299,17 +299,24 @@ private <I, O> void setConfiguration(Controller controller,
 		if (cfg == null) {
 			System.err
 					.println("Configuration is null. Runs the app with horizontal partitioning.");
-			Integer[] machineIds = new Integer[this.noOfnodes - 1];
-			for (int i = 0; i < machineIds.length; i++) {
-				machineIds[i] = i + 1;
-			}
-			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-					machineIds, stream, srcSink.first, srcSink.second);
-			app.newPartitionMap(partitionsMachineMap);
+			manualPartition(srcSink, stream, app);
+
 		} else
 			cfgManager.newConfiguration(cfg);
 	}
 
+	private <I, O> void manualPartition(
+			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
+			OneToOneElement<I, O> stream, StreamJitApp app) {
+		Integer[] machineIds = new Integer[this.noOfnodes - 1];
+		for (int i = 0; i < machineIds.length; i++) {
+			machineIds[i] = i + 1;
+		}
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
+				machineIds, stream, srcSink.first, srcSink.second);
+		app.newPartitionMap(partitionsMachineMap);
+	}
+
 	private <I, O> void setConstrains(Pair<Worker<I, ?>, Worker<?, O>> srcSink,
 			StreamJitApp app) {
 		// TODO: Copied form DebugStreamCompiler. Need to be verified for this

From f7486296a39dae3da45e6034d77fec672ebafe08 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Sep 2014 08:11:21 +0800
Subject: [PATCH 348/881] DSC.compile() refactored. setCfg refactored

---
 .../DistributedStreamCompiler.java            | 79 +++++++++++--------
 1 file changed, 47 insertions(+), 32 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index a2681f9c..1789fc5f 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -152,6 +152,25 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		return cs;
 	}
 
+	private <I, O> Configuration cfgFromFile(OneToOneElement<I, O> stream,
+			Controller controller, Configuration defaultCfg) {
+		Configuration cfg1 = readConfiguration(stream.getClass()
+				.getSimpleName());
+		if (cfg1 == null) {
+			controller.closeAll();
+			throw new IllegalConfigurationException();
+		} else if (!verifyCfg(defaultCfg, cfg1)) {
+			System.err
+					.println("Reading the configuration from configuration file");
+			System.err
+					.println("No matching between parameters in the read "
+							+ "configuration and parameters in the default configuration");
+			controller.closeAll();
+			throw new IllegalConfigurationException();
+		}
+		return cfg1;
+	}
+
 	/**
 	 * TODO: Need to check for other default subtypes of {@link OneToOneElement}
 	 * s. Now only checks for first generation children.
@@ -212,6 +231,18 @@ private Controller establishController() {
 		return partitionsMachineMap;
 	}
 
+	private <I, O> void manualPartition(
+			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
+			OneToOneElement<I, O> stream, StreamJitApp app) {
+		Integer[] machineIds = new Integer[this.noOfnodes - 1];
+		for (int i = 0; i < machineIds.length; i++) {
+			machineIds[i] = i + 1;
+		}
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
+				machineIds, stream, srcSink.first, srcSink.second);
+		app.newPartitionMap(partitionsMachineMap);
+	}
+
 	private Configuration readConfiguration(String simpeName) {
 		String name = String.format("%s.cfg", simpeName);
 		try {
@@ -275,46 +306,23 @@ private <I, O> void setConfiguration(Controller controller,
 			ConfigurationManager cfgManager, ConnectionManager conManager) {
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));
-		this.cfg = bf.getDefaultConfiguration(Workers
+		Configuration defaultCfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(srcSink.first));
-		if (GlobalConstants.tune == 0) {
-			Configuration cfg1 = readConfiguration(stream.getClass()
-					.getSimpleName());
-			if (cfg1 == null) {
-				controller.closeAll();
-				throw new IllegalConfigurationException();
-			} else if (!this.cfg.getParametersMap().keySet()
-					.equals(cfg1.getParametersMap().keySet())) {
-				System.err
-						.println("Reading the configuration from configuration file");
+
+		if (this.cfg != null) {
+			if (!verifyCfg(defaultCfg, this.cfg)) {
 				System.err
-						.println("No matching between parameters in the read "
+						.println("No matching between parameters in the passed "
 								+ "configuration and parameters in the default configuration");
 				controller.closeAll();
 				throw new IllegalConfigurationException();
 			}
-			this.cfg = cfg1;
-		}
-
-		if (cfg == null) {
-			System.err
-					.println("Configuration is null. Runs the app with horizontal partitioning.");
-			manualPartition(srcSink, stream, app);
-
+		} else if (GlobalConstants.tune == 0) {
+			this.cfg = cfgFromFile(stream, controller, defaultCfg);
 		} else
-			cfgManager.newConfiguration(cfg);
-	}
+			this.cfg = defaultCfg;
 
-	private <I, O> void manualPartition(
-			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
-			OneToOneElement<I, O> stream, StreamJitApp app) {
-		Integer[] machineIds = new Integer[this.noOfnodes - 1];
-		for (int i = 0; i < machineIds.length; i++) {
-			machineIds[i] = i + 1;
-		}
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-				machineIds, stream, srcSink.first, srcSink.second);
-		app.newPartitionMap(partitionsMachineMap);
+		cfgManager.newConfiguration(this.cfg);
 	}
 
 	private <I, O> void setConstrains(Pair<Worker<I, ?>, Worker<?, O>> srcSink,
@@ -331,6 +339,13 @@ private <I, O> void setConstrains(Pair<Worker<I, ?>, Worker<?, O>> srcSink,
 		app.constraints = constraints;
 	}
 
+	private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
+		if (defaultCfg.getParametersMap().keySet()
+				.equals(cfg.getParametersMap().keySet()))
+			return true;
+		return false;
+	}
+
 	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
 			OneToOneElement<I, O> stream) {
 		checkforDefaultOneToOneElement(stream);

From e3c4ad14080baa5f7743948f7ea9b015c5b05c04 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Sep 2014 09:14:00 +0800
Subject: [PATCH 349/881] Renaming : DrainDeadLockHandler->BufferCleaner

---
 .../distributed/node/BlobsManagerImpl.java    | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 284759ab..17693cb3 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -65,7 +65,7 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final ConnectionProvider conProvider;
 
-	private volatile DrainDeadLockHandler drainDeadLockHandler;
+	private volatile BufferCleaner bufferCleaner;
 
 	private final CTRLRDrainProcessor drainProcessor;
 
@@ -74,11 +74,11 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final StreamNode streamNode;
 
 	/**
-	 * if true {@link DrainDeadLockHandler} will be used to unlock the draining
+	 * if true {@link BufferCleaner} will be used to unlock the draining
 	 * time dead lock. Otherwise dynamic buffer will be used for local buffers
 	 * to handled drain time data growth.
 	 */
-	private final boolean useDrainDeadLockHandler;
+	private final boolean useBufferCleaner;
 
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
@@ -89,8 +89,8 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
-		this.drainDeadLockHandler = null;
-		this.useDrainDeadLockHandler = false;
+		this.bufferCleaner = null;
+		this.useBufferCleaner = false;
 		this.bufferManager = new LocalBufferManager(blobSet);
 
 		bufferManager.initialise();
@@ -156,8 +156,8 @@ public void stop() {
 		if (monBufs != null)
 			monBufs.stopMonitoring();
 
-		if (drainDeadLockHandler != null)
-			drainDeadLockHandler.stopit();
+		if (bufferCleaner != null)
+			bufferCleaner.stopit();
 	}
 
 	private void createBEs(ImmutableSet<Blob> blobSet) {
@@ -384,7 +384,7 @@ private void doDrain(boolean reqDrainData) {
 			// System.out.println("Blob " + blobID +
 			// "this.blob.drain(dcb); passed");
 
-			if (useDrainDeadLockHandler) {
+			if (useBufferCleaner) {
 				boolean isLastBlob = true;
 				for (BlobExecuter be : blobExecuters.values()) {
 					if (be.drainState == 0) {
@@ -393,10 +393,10 @@ private void doDrain(boolean reqDrainData) {
 					}
 				}
 
-				if (isLastBlob && drainDeadLockHandler == null) {
-					System.out.println("****Starting DrainDeadLockHandler***");
-					drainDeadLockHandler = new DrainDeadLockHandler();
-					drainDeadLockHandler.start();
+				if (isLastBlob && bufferCleaner == null) {
+					System.out.println("****Starting BufferCleaner***");
+					bufferCleaner = new BufferCleaner();
+					bufferCleaner.start();
 				}
 			}
 		}
@@ -458,8 +458,8 @@ private void drained() {
 				if (monBufs != null)
 					monBufs.stopMonitoring();
 
-				if (drainDeadLockHandler != null)
-					drainDeadLockHandler.stopit();
+				if (bufferCleaner != null)
+					bufferCleaner.stopit();
 
 			}
 			// printDrainedStatus();
@@ -747,12 +747,12 @@ public void run() {
 	 * @author sumanan
 	 * 
 	 */
-	private class DrainDeadLockHandler extends Thread {
+	private class BufferCleaner extends Thread {
 
 		final AtomicBoolean run;
 
-		private DrainDeadLockHandler() {
-			super("DrainDeadLockHandler");
+		private BufferCleaner() {
+			super("BufferCleaner");
 			this.run = new AtomicBoolean(true);
 		}
 
@@ -764,7 +764,7 @@ public void run() {
 			}
 
 			System.out
-					.println("DrainDeadLockHandler is goint to clean buffers...");
+					.println("BufferCleaner is goint to clean buffers...");
 			boolean areAllDrained = false;
 
 			while (run.get()) {

From 44fc75f2c7fb49832f6493d60a1856b6200bf97f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Sep 2014 12:30:21 +0800
Subject: [PATCH 350/881] DrainType introduced.

DrainType is introduced to avoid confusions on draining modes. Controller
decides appropriate draining type and send the type to SNs. SNs no more
checks GlobalConstants.useDrainData.
---
 .../impl/common/AbstractDrainer.java          | 23 ++++++----
 .../impl/concurrent/ConcurrentDrainer.java    |  3 +-
 .../impl/distributed/StreamJitAppManager.java |  7 ++--
 .../distributed/common/CTRLRDrainElement.java | 42 +++++++++++++++----
 .../distributed/node/BlobsManagerImpl.java    | 30 +++++--------
 .../runtimer/DistributedDrainer.java          |  5 ++-
 6 files changed, 67 insertions(+), 43 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index f11cc29e..a762ba82 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -2,10 +2,8 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -37,6 +35,7 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
@@ -290,6 +289,7 @@ public final DrainData getDrainData() {
 		// }
 		return drainData;
 	}
+
 	/**
 	 * logs the size of the drain data on each channel for every draining and
 	 * writes down the statistics into a file.
@@ -390,13 +390,8 @@ public final void awaitDrained(long timeout, TimeUnit unit)
 	/**
 	 * Once a {@link BlobNode}'s all preconditions are satisfied for draining,
 	 * blob node will call this function drain the blob.
-	 * 
-	 * @param blobID
-	 * @param isFinal
-	 *            : whether the draining is the final draining or intermediate
-	 *            draining. Set to true for semi final case.
 	 */
-	protected abstract void drain(Token blobID, boolean isFinal);
+	protected abstract void drain(Token blobID, DrainType drainType);
 
 	/**
 	 * {@link AbstractDrainer} will call this function after the corresponding
@@ -734,7 +729,17 @@ private void drain() {
 				throw new IllegalStateException(
 						"Drain of this blobNode has already been called");
 			}
-			drainer.drain(blobID, drainer.state == DrainerState.FINAL);
+
+			DrainType drainType;
+			if (GlobalConstants.useDrainData)
+				if (drainer.state == DrainerState.FINAL)
+					drainType = DrainType.FINAL;
+				else
+					drainType = DrainType.INTERMEDIATE;
+			else
+				drainType = DrainType.DISCARD;
+
+			drainer.drain(blobID, drainType);
 
 			// TODO: Verify the waiting time is reasonable.
 			if (GlobalConstants.needDrainDeadlockHandler)
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
index 7b7c104a..614a3dea 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
@@ -11,6 +11,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.BlobThread;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
 /**
@@ -45,7 +46,7 @@ protected void drainingDone(boolean isFinal) {
 	}
 
 	@Override
-	protected void drain(Token blobID, boolean isFinal) {
+	protected void drain(Token blobID, DrainType drainType) {
 		Blob blob = blobMap.get(blobID);
 		checkNotNull(blob);
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 82a9eee8..00f7a9ee 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -23,6 +23,7 @@
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Command;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
@@ -118,14 +119,14 @@ public AppStatusProcessor appStatusProcessor() {
 		return apStsPro;
 	}
 
-	public void drain(Token blobID, boolean isFinal) {
+	public void drain(Token blobID, DrainType drainType) {
 		// System.out.println("Drain requested to blob " + blobID);
 		if (!app.blobtoMachineMap.containsKey(blobID))
 			throw new IllegalArgumentException(blobID
 					+ " not found in the blobtoMachineMap");
 		int nodeID = app.blobtoMachineMap.get(blobID);
-		controller
-				.send(nodeID, new CTRLRDrainElement.DoDrain(blobID, !isFinal));
+		controller.send(nodeID,
+				new CTRLRDrainElement.DoDrain(blobID, drainType));
 	}
 
 	public void drainingFinished(boolean isFinal) {
diff --git a/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java b/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
index 4d836a29..893518e5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
@@ -4,7 +4,7 @@
 
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
@@ -60,12 +60,7 @@ public void process(CTRLRDrainProcessor dp) {
 	public static final class DoDrain extends CTRLRDrainElement {
 		private static final long serialVersionUID = 1L;
 
-		/**
-		 * Instead of sending another object to get the {@link DrainData},
-		 * {@link Controller} can set this flag to get the drain data once
-		 * draining is done.
-		 */
-		public final boolean reqDrainData;
+		public final DrainType drainType;
 
 		/**
 		 * Identifies the blob. Since {@link Blob}s do not have an unique
@@ -74,9 +69,9 @@ public static final class DoDrain extends CTRLRDrainElement {
 		 */
 		public final Token blobID;
 
-		public DoDrain(Token blobID, boolean reqDrainData) {
+		public DoDrain(Token blobID, DrainType drainType) {
 			this.blobID = blobID;
-			this.reqDrainData = reqDrainData;
+			this.drainType = drainType;
 		}
 
 		@Override
@@ -98,4 +93,33 @@ public interface CTRLRDrainProcessor {
 
 		public void process(DoDrain drain);
 	}
+
+	/**
+	 * Three types of draining are possible.
+	 * <ol>
+	 * <li>drainType = 1 : Final draining. No drain data. All {@link Blob}s are
+	 * expected to run and finish all data.
+	 * <li>drainType = 2 : Intermediate draining. Drain data is required in this
+	 * mode. {@link BoundaryInputChannel}s may create extra buffer and put all
+	 * unconsumed data, and finally send this drain data to the
+	 * {@link Controller} for reconfiguration.
+	 * <li>drainType = 3 : Discard all unconsumed data. This is useful, if we
+	 * don't care about the data while tuning for performance.
+	 * </ol>
+	 * 
+	 * See the same description at {@link BoundaryInputChannel#stop(int)}.
+	 * 
+	 */
+	public enum DrainType {
+		FINAL(1), INTERMEDIATE(2), DISCARD(3);
+		private final int code;
+
+		DrainType(int code) {
+			this.code = code;
+		}
+
+		public int toint() {
+			return code;
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 17693cb3..f6a7d25b 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -34,11 +34,11 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
@@ -101,10 +101,10 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 	/**
 	 * Drain the blob identified by the token.
 	 */
-	public void drain(Token blobID, boolean reqDrainData) {
+	public void drain(Token blobID, DrainType drainType) {
 		for (BlobExecuter be : blobExecuters.values()) {
 			if (be.getBlobID().equals(blobID)) {
-				be.doDrain(reqDrainData);
+				be.doDrain(drainType);
 				return;
 			}
 		}
@@ -267,7 +267,7 @@ private class BlobExecuter {
 
 		private final BoundaryOutputChannelManager outChnlManager;
 
-		private boolean reqDrainData;
+		private DrainType drainType;
 
 		private BlobExecuter(Token t, Blob blob,
 				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
@@ -352,28 +352,20 @@ private ImmutableMap<Token, Buffer> buildBufferMap() {
 			return bufferMapBuilder.build();
 		}
 
-		private void doDrain(boolean reqDrainData) {
+		private void doDrain(DrainType drainType) {
 			// System.out.println("Blob " + blobID + "is doDrain");
-			this.reqDrainData = reqDrainData;
+			this.drainType = drainType;
 			drainState = 1;
 
-			int stopType;
 			// TODO: [2014-02-05] rearranged this order to call stop(3)
 			// whenever GlobalConstants.useDrainData is false irrespective
 			// of reqDrainData.
-			if (GlobalConstants.useDrainData)
-				if (!this.reqDrainData)
-					stopType = 1;
-				else
-					stopType = 2;
-			else
-				stopType = 3;
 
-			inChnlManager.stop(stopType);
+			inChnlManager.stop(drainType.toint());
 			// TODO: [2014-03-14] I commented following line to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
 			// [2014-09-17] Lets waitToStop() if drain data is required.
-			if (stopType != 3)
+			if (drainType != DrainType.DISCARD)
 				inChnlManager.waitToStop();
 
 			if (this.blob != null) {
@@ -412,7 +404,7 @@ private void drained() {
 				bt.requestStop();
 			}
 
-			outChnlManager.stop(!this.reqDrainData);
+			outChnlManager.stop(drainType == DrainType.FINAL);
 			outChnlManager.waitToStop();
 
 			if (drainState > 3)
@@ -427,7 +419,7 @@ private void drained() {
 			}
 			// System.out.println("Blob " + blobID + "is drained at mid");
 
-			if (GlobalConstants.useDrainData && this.reqDrainData) {
+			if (drainType == DrainType.INTERMEDIATE) {
 				SNMessageElement me;
 				if (crashed.get())
 					me = getEmptyDrainData();
@@ -698,7 +690,7 @@ private class CTRLRDrainProcessorImpl implements CTRLRDrainProcessor {
 
 		@Override
 		public void process(DoDrain drain) {
-			drain(drain.blobID, drain.reqDrainData);
+			drain(drain.blobID, drain.drainType);
 		}
 
 		@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
index ff43373d..86fab2a5 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
@@ -3,6 +3,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 
 /**
  * @author Sumanan sumanan@mit.edu
@@ -25,8 +26,8 @@ protected void drainingDone(boolean isFinal) {
 	}
 
 	@Override
-	protected void drain(Token blobID, boolean isFinal) {
-		manager.drain(blobID, isFinal);
+	protected void drain(Token blobID, DrainType drainType) {
+		manager.drain(blobID, drainType);
 	}
 
 	@Override

From a4b27579c1155724114f6fc158851925b0c5a40e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Sep 2014 16:39:39 +0800
Subject: [PATCH 351/881] BufferCleaner makes drainData

Depends on the options, bufferCleaner copies the extra data from the local
buffer to draindata.
---
 .../distributed/node/BlobsManagerImpl.java    | 135 +++++++++++++++---
 1 file changed, 115 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f6a7d25b..9d32bed1 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -4,6 +4,8 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -74,9 +76,9 @@ public class BlobsManagerImpl implements BlobsManager {
 	private final StreamNode streamNode;
 
 	/**
-	 * if true {@link BufferCleaner} will be used to unlock the draining
-	 * time dead lock. Otherwise dynamic buffer will be used for local buffers
-	 * to handled drain time data growth.
+	 * if true {@link BufferCleaner} will be used to unlock the draining time
+	 * dead lock. Otherwise dynamic buffer will be used for local buffers to
+	 * handled drain time data growth.
 	 */
 	private final boolean useBufferCleaner;
 
@@ -357,10 +359,6 @@ private void doDrain(DrainType drainType) {
 			this.drainType = drainType;
 			drainState = 1;
 
-			// TODO: [2014-02-05] rearranged this order to call stop(3)
-			// whenever GlobalConstants.useDrainData is false irrespective
-			// of reqDrainData.
-
 			inChnlManager.stop(drainType.toint());
 			// TODO: [2014-03-14] I commented following line to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
@@ -376,7 +374,7 @@ private void doDrain(DrainType drainType) {
 			// System.out.println("Blob " + blobID +
 			// "this.blob.drain(dcb); passed");
 
-			if (useBufferCleaner) {
+			if (useBufferCleaner && drainType != DrainType.FINAL) {
 				boolean isLastBlob = true;
 				for (BlobExecuter be : blobExecuters.values()) {
 					if (be.drainState == 0) {
@@ -387,7 +385,8 @@ private void doDrain(DrainType drainType) {
 
 				if (isLastBlob && bufferCleaner == null) {
 					System.out.println("****Starting BufferCleaner***");
-					bufferCleaner = new BufferCleaner();
+					bufferCleaner = new BufferCleaner(
+							drainType == DrainType.INTERMEDIATE);
 					bufferCleaner.start();
 				}
 			}
@@ -473,6 +472,7 @@ private DrainedData getDrainData() {
 
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
+
 			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
 					.inputChannelsMap();
 
@@ -492,14 +492,8 @@ private DrainedData getDrainData() {
 					inputDataBuilder.put(t, draindata);
 				}
 
-				// TODO: Unnecessary data copy. Optimise this.
 				else {
-					Buffer buf = bufferMap.get(t);
-					Object[] bufArray = new Object[buf.size()];
-					buf.readAll(bufArray);
-					assert buf.size() == 0 : String.format(
-							"buffer size is %d. But 0 is expected", buf.size());
-					inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
+					unprocessedDataFromLocalBuffer(inputDataBuilder, t);
 				}
 			}
 
@@ -521,6 +515,23 @@ private DrainedData getDrainData() {
 					inputDataBuilder.build(), outputDataBuilder.build());
 		}
 
+		// TODO: Unnecessary data copy. Optimise this.
+		private void unprocessedDataFromLocalBuffer(
+				ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder,
+				Token t) {
+			Object[] bufArray;
+			if (bufferCleaner == null) {
+				Buffer buf = bufferMap.get(t);
+				bufArray = new Object[buf.size()];
+				buf.readAll(bufArray);
+				assert buf.size() == 0 : String.format(
+						"buffer size is %d. But 0 is expected", buf.size());
+			} else {
+				bufArray = bufferCleaner.copiedBuffer(t);
+			}
+			inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
+		}
+
 		private DrainedData getEmptyDrainData() {
 			drainState = 5;
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
@@ -743,9 +754,20 @@ private class BufferCleaner extends Thread {
 
 		final AtomicBoolean run;
 
-		private BufferCleaner() {
+		final boolean needToCopyDrainData;
+
+		final Map<Token, List<Object[]>> newlocalBufferMap;
+
+		private BufferCleaner(boolean needToCopyDrainData) {
 			super("BufferCleaner");
+			System.out.println("Buffer Cleaner : needToCopyDrainData == "
+					+ needToCopyDrainData);
 			this.run = new AtomicBoolean(true);
+			this.needToCopyDrainData = needToCopyDrainData;
+			if (needToCopyDrainData)
+				newlocalBufferMap = new HashMap<>();
+			else
+				newlocalBufferMap = null;
 		}
 
 		public void run() {
@@ -755,14 +777,23 @@ public void run() {
 				return;
 			}
 
-			System.out
-					.println("BufferCleaner is goint to clean buffers...");
+			System.out.println("BufferCleaner is going to clean buffers...");
 			boolean areAllDrained = false;
 
 			while (run.get()) {
-				areAllDrained = cleanAllBuffers();
+				if (needToCopyDrainData)
+					areAllDrained = copyLocalBuffers();
+				else
+					areAllDrained = cleanAllBuffers();
+
 				if (areAllDrained)
 					break;
+
+				try {
+					Thread.sleep(1000);
+				} catch (InterruptedException e) {
+					break;
+				}
 			}
 		}
 
@@ -804,10 +835,74 @@ private void clean(Buffer b, Token t) {
 			b.readAll(obArray);
 		}
 
+		/**
+		 * Copy only the local buffers into a new large buffer to make the
+		 * blocked blob to progress. This copied buffer can be sent to
+		 * controller as a drain data.
+		 */
+		private boolean copyLocalBuffers() {
+			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
+					.localBufferMap();
+			boolean areAllDrained = true;
+			for (BlobExecuter be : blobExecuters.values()) {
+				if (be.drainState == 1 || be.drainState == 2) {
+					// System.out.println(be.blobID + " is not drained");
+					areAllDrained = false;
+					for (Token t : be.blob.getOutputs()) {
+						if (localBufferMap.containsKey(t)) {
+							Buffer b = be.bufferMap.get(t);
+							copy(b, t);
+						}
+					}
+				}
+			}
+			return areAllDrained;
+		}
+
+		private void copy(Buffer b, Token t) {
+			int size = b.size();
+			if (size == 0)
+				return;
+
+			if (!newlocalBufferMap.containsKey(t)) {
+				newlocalBufferMap.put(t, new LinkedList<Object[]>());
+			}
+
+			List<Object[]> list = newlocalBufferMap.get(t);
+			Object[] bufArray = new Object[size];
+			b.readAll(bufArray);
+			assert b.size() == 0 : String.format(
+					"buffer size is %d. But 0 is expected", b.size());
+			list.add(bufArray);
+		}
+
 		public void stopit() {
 			this.run.set(false);
 			this.interrupt();
 		}
+
+		public Object[] copiedBuffer(Token t) {
+			assert needToCopyDrainData : "BufferCleaner is not in buffer copy mode";
+			copy(bufferManager.localBufferMap().get(t), t);
+			List<Object[]> list = newlocalBufferMap.get(t);
+			if (list.size() == 0)
+				return new Object[0];
+			else if (list.size() == 1)
+				return list.get(0);
+
+			int size = 0;
+			for (Object[] array : list) {
+				size += array.length;
+			}
+
+			int destPos = 0;
+			Object[] mergedArray = new Object[size];
+			for (Object[] array : list) {
+				System.arraycopy(array, 0, mergedArray, destPos, array.length);
+				destPos += array.length;
+			}
+			return mergedArray;
+		}
 	}
 
 	private static int count = 0;

From a28d93e134d1663ad4a920e6aca0a661516ec3d4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Sep 2014 22:17:46 +0800
Subject: [PATCH 352/881] BugFix : list can become null.

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 9d32bed1..2e08b16f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -885,7 +885,9 @@ public Object[] copiedBuffer(Token t) {
 			assert needToCopyDrainData : "BufferCleaner is not in buffer copy mode";
 			copy(bufferManager.localBufferMap().get(t), t);
 			List<Object[]> list = newlocalBufferMap.get(t);
-			if (list.size() == 0)
+			if (list == null)
+				return new Object[0];
+			else if (list.size() == 0)
 				return new Object[0];
 			else if (list.size() == 1)
 				return list.get(0);

From 84b22000390bc1d09dcc47f775d7916319e08649 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Sep 2014 06:13:11 +0800
Subject: [PATCH 353/881] Renaming:LocalBufferManager->SNLocalBufferManager

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java | 4 ++--
 .../mit/streamjit/impl/distributed/node/BufferManager.java    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2e08b16f..f5b52f7d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -45,7 +45,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.Utils;
-import edu.mit.streamjit.impl.distributed.node.BufferManager.LocalBufferManager;
+import edu.mit.streamjit.impl.distributed.node.BufferManager.SNLocalBufferManager;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
@@ -93,7 +93,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
 		this.bufferCleaner = null;
 		this.useBufferCleaner = false;
-		this.bufferManager = new LocalBufferManager(blobSet);
+		this.bufferManager = new SNLocalBufferManager(blobSet);
 
 		bufferManager.initialise();
 		if (bufferManager.isbufferSizesReady())
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index cb723c12..14212425 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -147,8 +147,8 @@ protected final void createLocalBuffers() {
 	 * Calculates buffer sizes locally at {@link StreamNode} side. No central
 	 * calculation involved.
 	 */
-	public static class LocalBufferManager extends AbstractBufferManager {
-		public LocalBufferManager(Set<Blob> blobSet) {
+	public static class SNLocalBufferManager extends AbstractBufferManager {
+		public SNLocalBufferManager(Set<Blob> blobSet) {
 			super(blobSet);
 		}
 

From 8552811c6531566efe018d43a36bdc53a4b6a3c7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Sep 2014 08:24:34 +0800
Subject: [PATCH 354/881] A flag monitorBuffers added

The boolean variable monitorBuffers enables or disables buffer size monitoring.
---
 .../distributed/node/BlobsManagerImpl.java    | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f5b52f7d..102ecad0 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -67,11 +67,11 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final ConnectionProvider conProvider;
 
-	private volatile BufferCleaner bufferCleaner;
+	private volatile BufferCleaner bufferCleaner = null;
 
 	private final CTRLRDrainProcessor drainProcessor;
 
-	private MonitorBuffers monBufs;
+	private MonitorBuffers monBufs = null;
 
 	private final StreamNode streamNode;
 
@@ -80,7 +80,13 @@ public class BlobsManagerImpl implements BlobsManager {
 	 * dead lock. Otherwise dynamic buffer will be used for local buffers to
 	 * handled drain time data growth.
 	 */
-	private final boolean useBufferCleaner;
+	private final boolean useBufferCleaner = false;
+
+	/**
+	 * if true {@link MonitorBuffers} will be started to log the buffer sizes
+	 * periodically.
+	 */
+	private final boolean monitorBuffers = false;
 
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
@@ -91,8 +97,6 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
-		this.bufferCleaner = null;
-		this.useBufferCleaner = false;
 		this.bufferManager = new SNLocalBufferManager(blobSet);
 
 		bufferManager.initialise();
@@ -138,13 +142,11 @@ public void start() {
 		for (BlobExecuter be : blobExecuters.values())
 			be.start();
 
-		// if (monBufs == null) {
-		// // System.out.println("Creating new MonitorBuffers");
-		// monBufs = new MonitorBuffers();
-		// monBufs.start();
-		// } else
-		// System.err
-		// .println("Mon buffer is not null. Check the logic for bug");
+		if (monitorBuffers && monBufs == null) {
+			// System.out.println("Creating new MonitorBuffers");
+			monBufs = new MonitorBuffers();
+			monBufs.start();
+		}
 	}
 
 	/**

From e8f49404fa46fa9ecb541fb871e69e537c891e9c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 26 Sep 2014 12:10:25 +0800
Subject: [PATCH 355/881] MonBuffer prints discriptive details about buffers

MonitorBuffer prints discriptive messages such as whether a buffer is full or
not, whether a blob has enough input data to fire, whether a blob has enough
room to write its output and etc.
---
 .../distributed/node/BlobsManagerImpl.java    | 44 ++++++++++++++++---
 1 file changed, 38 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 102ecad0..e7e3aad4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -945,25 +945,57 @@ public void run() {
 					try {
 						Thread.sleep(sleepTime);
 					} catch (InterruptedException e) {
+						break;
 					}
+
+					if (stopFlag.get())
+						break;
+
 					if (blobExecuters == null) {
-						writter.write("Buffer map is null...\n");
+						writter.write("blobExecuters are null...\n");
 						continue;
 					}
+
 					writter.write("----------------------------------\n");
 					for (BlobExecuter be : blobExecuters.values()) {
+						writter.write("Status of blob " + be.blobID.toString()
+								+ "\n");
+
 						if (be.bufferMap == null) {
 							writter.write("Buffer map is null...\n");
 							continue;
 						}
+
 						if (stopFlag.get())
 							break;
 
-						for (Map.Entry<Token, Buffer> en : be.bufferMap
-								.entrySet()) {
-							writter.write(en.getKey() + " - "
-									+ en.getValue().size());
-							writter.write('\n');
+						writter.write("Input channel details\n");
+						for (Token t : be.inChnlManager.inputChannelsMap()
+								.keySet()) {
+							Buffer b = be.bufferMap.get(t);
+							if (b == null)
+								continue;
+							int min = be.blob.getMinimumBufferCapacity(t);
+							int size = b.size();
+							String status = size > min ? "Firable"
+									: "NOT firable";
+							writter.write(t.toString() + "\tMin - " + min
+									+ ",\tSize - " + size + "\t" + status
+									+ "\n");
+						}
+
+						writter.write("Output channel details\n");
+						for (Token t : be.outChnlManager.outputChannelsMap()
+								.keySet()) {
+							Buffer b = be.bufferMap.get(t);
+							if (b == null)
+								continue;
+							int min = be.blob.getMinimumBufferCapacity(t);
+							int size = b.size();
+							String status = size == 0 ? "Empty" : "NOT Empty";
+							writter.write(t.toString() + "\tMin - " + min
+									+ ",\tSize - " + size + "\t" + status
+									+ "\n");
 						}
 					}
 					writter.write("----------------------------------\n");

From a7f1cf676d3ac6375e4ffd5a5b39f1eee0f959f5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 26 Sep 2014 19:05:41 +0800
Subject: [PATCH 356/881] Refactored : MonitorBuffer.run()

Moved repeating code snippet to a new private method write().
---
 .../distributed/node/BlobsManagerImpl.java    | 55 ++++++++++---------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index e7e3aad4..05364895 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -970,33 +970,10 @@ public void run() {
 							break;
 
 						writter.write("Input channel details\n");
-						for (Token t : be.inChnlManager.inputChannelsMap()
-								.keySet()) {
-							Buffer b = be.bufferMap.get(t);
-							if (b == null)
-								continue;
-							int min = be.blob.getMinimumBufferCapacity(t);
-							int size = b.size();
-							String status = size > min ? "Firable"
-									: "NOT firable";
-							writter.write(t.toString() + "\tMin - " + min
-									+ ",\tSize - " + size + "\t" + status
-									+ "\n");
-						}
+						write(be, writter, true);
 
 						writter.write("Output channel details\n");
-						for (Token t : be.outChnlManager.outputChannelsMap()
-								.keySet()) {
-							Buffer b = be.bufferMap.get(t);
-							if (b == null)
-								continue;
-							int min = be.blob.getMinimumBufferCapacity(t);
-							int size = b.size();
-							String status = size == 0 ? "Empty" : "NOT Empty";
-							writter.write(t.toString() + "\tMin - " + min
-									+ ",\tSize - " + size + "\t" + status
-									+ "\n");
-						}
+						write(be, writter, false);
 					}
 					writter.write("----------------------------------\n");
 					writter.flush();
@@ -1017,6 +994,34 @@ public void run() {
 			}
 		}
 
+		public void write(BlobExecuter be, FileWriter writter, boolean isIn)
+				throws IOException {
+			ImmutableSet<Token> tokenSet;
+			if (isIn)
+				tokenSet = be.inChnlManager.inputChannelsMap().keySet();
+			else
+				tokenSet = be.outChnlManager.outputChannelsMap().keySet();
+
+			for (Token t : tokenSet) {
+				Buffer b = be.bufferMap.get(t);
+				if (b == null)
+					continue;
+				int min = Integer.MAX_VALUE;
+				// BE sets blob to null after the drained().
+				if (be.blob != null)
+					min = be.blob.getMinimumBufferCapacity(t);
+
+				int availableResource = isIn ? b.size() : b.capacity()
+						- b.size();
+
+				String status = availableResource > min ? "Firable"
+						: "NOT firable";
+				writter.write(t.toString() + "\tMin - " + min
+						+ ",\tAvailableResource - " + availableResource + "\t"
+						+ status + "\n");
+			}
+		}
+
 		public void stopMonitoring() {
 			// System.out.println("MonitorBuffers: Stop monitoring");
 			stopFlag.set(true);

From 40fdb88241cdd4c207a1c5b12d078c874f84048f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 26 Sep 2014 19:39:07 +0800
Subject: [PATCH 357/881] Bugfix: availableResource >= min

is enough condition to fire.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 05364895..f63de1cb 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -1014,7 +1014,7 @@ public void write(BlobExecuter be, FileWriter writter, boolean isIn)
 				int availableResource = isIn ? b.size() : b.capacity()
 						- b.size();
 
-				String status = availableResource > min ? "Firable"
+				String status = availableResource >= min ? "Firable"
 						: "NOT firable";
 				writter.write(t.toString() + "\tMin - " + min
 						+ ",\tAvailableResource - " + availableResource + "\t"

From 2149298012769f31579b17730f301c968f946dca Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 27 Sep 2014 12:18:25 +0800
Subject: [PATCH 358/881] MonitorBuffer.tokenSet() added

As BlobExecuter sets blob to null after the drain(), We cannot always get
the token set from blob.
---
 .../distributed/node/BlobsManagerImpl.java    | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f63de1cb..cf835445 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -994,14 +994,9 @@ public void run() {
 			}
 		}
 
-		public void write(BlobExecuter be, FileWriter writter, boolean isIn)
+		private void write(BlobExecuter be, FileWriter writter, boolean isIn)
 				throws IOException {
-			ImmutableSet<Token> tokenSet;
-			if (isIn)
-				tokenSet = be.inChnlManager.inputChannelsMap().keySet();
-			else
-				tokenSet = be.outChnlManager.outputChannelsMap().keySet();
-
+			Set<Token> tokenSet = tokenSet(be, isIn);
 			for (Token t : tokenSet) {
 				Buffer b = be.bufferMap.get(t);
 				if (b == null)
@@ -1022,6 +1017,23 @@ public void write(BlobExecuter be, FileWriter writter, boolean isIn)
 			}
 		}
 
+		private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
+			Set<Token> tokenSet;
+			// BE sets blob to null after the drained().
+			if (be.blob == null) {
+				if (isIn)
+					tokenSet = be.inChnlManager.inputChannelsMap().keySet();
+				else
+					tokenSet = be.outChnlManager.outputChannelsMap().keySet();
+			} else {
+				if (isIn)
+					tokenSet = be.blob.getInputs();
+				else
+					tokenSet = be.blob.getOutputs();
+			}
+			return tokenSet;
+		}
+
 		public void stopMonitoring() {
 			// System.out.println("MonitorBuffers: Stop monitoring");
 			stopFlag.set(true);

From 46c968e1691bb787017baa4cb820392c960fbde6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 27 Sep 2014 12:27:44 +0800
Subject: [PATCH 359/881] Refactored : printDrainDataStats() added.

BlobExecuter.getDrainData() is refactored and a new method printDrainDataStats()
added.
---
 .../distributed/node/BlobsManagerImpl.java    | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index cf835445..5cf535c4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -461,16 +461,10 @@ private void drained() {
 		private DrainedData getDrainData() {
 			if (this.blob == null)
 				return getEmptyDrainData();
-			// System.out.println("**********************************");
+
 			DrainData dd = blob.getDrainData();
 			drainState = 5;
-
-			if (dd != null) {
-				// for (Token t : dd.getData().keySet()) {
-				// System.out.println("From Blob: " + t.toString() + " - "
-				// + dd.getData().get(t).size());
-				// }
-			}
+			// printDrainDataStats(dd);
 
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
@@ -566,6 +560,16 @@ private String getName(Blob blob) {
 			return sb.toString();
 		}
 
+		private void printDrainDataStats(DrainData dd) {
+			System.out.println("**********printDrainDataStats*************");
+			if (dd != null) {
+				for (Token t : dd.getData().keySet()) {
+					System.out.println("From Blob: " + t.toString() + " - "
+							+ dd.getData().get(t).size());
+				}
+			}
+		}
+
 		private void start() {
 			outChnlManager.waitToStart();
 			inChnlManager.waitToStart();

From fcd6bb086f774abfb2c07be1c8f952ae139eaaa4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 29 Sep 2014 10:35:22 +0800
Subject: [PATCH 360/881] changeMultiplierVal() added to configurationEditor

This method reads a configuration file and updates it's multiplier value with
a new value.
---
 .../impl/common/ConfigurationEditor.java      | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index cbb5d38b..fc9c27c6 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -37,10 +37,30 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		generate1(new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
-		edit1(name, noofwrks);
+		// generate1(new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
+		// edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
+		changeMultiplierVal();
+	}
+
+	/**
+	 * Reads a configuration and changes its multiplier value.
+	 */
+	private static void changeMultiplierVal() {
+		Configuration config = readConfiguration("final_NestedSplitJoinCore");
+		Configuration.Builder builder = Configuration.builder(config);
+		IntParameter mulParam = config.getParameter("multiplier",
+				IntParameter.class);
+		if (mulParam != null) {
+			System.out.println("Multiplier values is " + mulParam.getValue());
+			builder.removeParameter(mulParam.getName());
+		}
+
+		IntParameter newMulParam = new IntParameter("multiplier", 1, 100, 100);
+		builder.addParameter(newMulParam);
+		saveConfg(builder.build(), 444, "final_NestedSplitJoinCore");
+
 	}
 
 	private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {

From 108797acd04298578110eddaa7c9b7123d2aa2ad Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Sep 2014 12:26:56 +0800
Subject: [PATCH 361/881] An interface "LocalBuffer" added

LocalBuffer connects up blob and down blob where both are running at same
StreamNode. In this case, we can simply use any Buffer implementation. But at
the draining time blobs write large amount of data, which blobs buffered inside
during the init schedule, and limited buffer size causes deadlock.
Implementations of the LocalBuffer are supposed to understand drainingStarted
event and increase the buffer size smartly to avoid deadlock situation.
see Deadlock 5.

LocalBuffer1, an implementation of LocalBuffer, has also been added.
---
 .../impl/distributed/node/LocalBuffer.java    | 219 ++++++++++++++++++
 1 file changed, 219 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
new file mode 100644
index 00000000..316424aa
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
@@ -0,0 +1,219 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.util.ConstructorSupplier;
+import edu.mit.streamjit.util.ReflectionUtils;
+
+/**
+ * {@link LocalBuffer} connects up blob and down blob where both are running to
+ * same StreamNode. In this case, we can simply use any {@link Buffer}
+ * implementation to connect up blob and down blob. But at the draining time
+ * blobs write large amount of data, which blobs buffered inside during the init
+ * schedule, and limited buffer size causes deadlock. Implementations of the
+ * {@link LocalBuffer} are supposed to understand drainingStarted event and
+ * increase the buffer size smartly to avoid deadlock situation. see Deadlock 5.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Sept 23, 2014
+ */
+public interface LocalBuffer extends Buffer {
+
+	public void drainingStarted();
+
+	/**
+	 * Modified version of {@link DynamicBufferManager#DynamicBuffer}. Instead
+	 * of dynamically increase the buffer sizes, this implementation creates an
+	 * additional buffer to unlock the draining time deadlock.
+	 * 
+	 * 
+	 * Blobs write more than expected amount of data during the draining time
+	 * and, sometimes the output buffers become full forever at the draining
+	 * time and blobs spin on write() forever. This implementation creates new a
+	 * supplied buffer in order to avoid dead locks during draining time.
+	 * 
+	 * <p>
+	 * Determining whether buffer fullness is due to deadlock situation or the
+	 * current blob is executing on a faster node than the down stream blob is
+	 * little tricky.
+	 * </p>
+	 * 
+	 * <p>
+	 * TODO: {@link ConstructorSupplier} can be reused here to instantiate the
+	 * buffer instances if we make {@link ConstructorSupplier}.arguments not
+	 * final.
+	 * </p>
+	 * 
+	 */
+	public class LocalBuffer1 implements LocalBuffer {
+
+		private final int capacityPos;
+
+		private final Constructor<? extends Buffer> cons;
+
+		private final Buffer defaultBuffer;
+
+		private volatile Buffer drainBuffer;
+
+		/**
+		 * Minimum time gap between the last successful write and the current
+		 * time in order to consider the option of doubling the buffer
+		 */
+		private final long gap;
+
+		private volatile boolean hasDrainingStarted;
+
+		private final List<?> initialArguments;
+
+		private final int initialCapacity;
+
+		/**
+		 * Every successful write operation should update this time.
+		 */
+		private long lastWrittenTime;
+
+		private final String name;
+
+		private Buffer writeBuffer;
+
+		public LocalBuffer1(String name, Class<? extends Buffer> bufferClass,
+				List<?> initialArguments, int initialCapacity, int capacityPos) {
+			this.name = name;
+			this.initialArguments = initialArguments;
+			this.initialCapacity = initialCapacity;
+			this.capacityPos = capacityPos;
+			Constructor<? extends Buffer> con = null;
+			try {
+				con = ReflectionUtils.findConstructor(bufferClass,
+						initialArguments);
+			} catch (NoSuchMethodException e1) {
+				e1.printStackTrace();
+			}
+			this.cons = con;
+			this.defaultBuffer = getNewBuffer(initialCapacity);
+			this.writeBuffer = defaultBuffer;
+			this.gap = 10_000_000_000l; // 10s
+			hasDrainingStarted = false;
+			lastWrittenTime = 0;
+		}
+
+		@Override
+		public int capacity() {
+			int cap = drainBuffer == null ? defaultBuffer.capacity()
+					: drainBuffer.capacity() + defaultBuffer.capacity();
+			return cap;
+		}
+
+		@Override
+		public void drainingStarted() {
+			hasDrainingStarted = true;
+		}
+
+		@Override
+		public Object read() {
+			Object o = defaultBuffer.read();
+			return o;
+		}
+
+		@Override
+		public int read(Object[] data, int offset, int length) {
+			int ret = defaultBuffer.read(data, offset, length);
+			return ret;
+		}
+
+		@Override
+		public boolean readAll(Object[] data) {
+			boolean ret = defaultBuffer.readAll(data);
+			return ret;
+		}
+
+		@Override
+		public boolean readAll(Object[] data, int offset) {
+			boolean ret = defaultBuffer.readAll(data, offset);
+			return ret;
+		}
+
+		@Override
+		public int size() {
+			int size = drainBuffer == null ? defaultBuffer.size() : drainBuffer
+					.size() + defaultBuffer.size();
+			return size;
+		}
+
+		@Override
+		public boolean write(Object t) {
+			boolean ret = writeBuffer.write(t);
+			if (!ret)
+				writeFailed();
+			else if (lastWrittenTime != 0)
+				lastWrittenTime = 0;
+			return ret;
+		}
+
+		@Override
+		public int write(Object[] data, int offset, int length) {
+			int written = writeBuffer.write(data, offset, length);
+			if (written == 0)
+				writeFailed();
+			else if (lastWrittenTime != 0)
+				lastWrittenTime = 0;
+			return written;
+		}
+
+		private void createDrainBuffer() {
+			assert drainBuffer == null : "drainBuffer has already been created.";
+			int newCapacity = 2 * defaultBuffer.capacity();
+			System.out
+					.println(String
+							.format("%s : Creating drain buffer: defaultBufferCapacity - %d, drainBufferCapacity - %d",
+									name, initialCapacity, newCapacity));
+			drainBuffer = getNewBuffer(newCapacity);
+			this.writeBuffer = drainBuffer;
+			lastWrittenTime = 0;
+		}
+
+		private List<?> getArguments(int newCapacity) {
+			List<Object> newArgs = new ArrayList<>(initialArguments.size());
+			for (int i = 0; i < initialArguments.size(); i++) {
+				if (i == capacityPos)
+					newArgs.add(newCapacity);
+				else
+					newArgs.add(initialArguments.get(i));
+			}
+			return newArgs;
+		}
+
+		private Buffer getNewBuffer(int newCapacity) {
+			Buffer buffer;
+			try {
+				buffer = cons.newInstance(getArguments(newCapacity).toArray());
+				return buffer;
+			} catch (InstantiationException | IllegalAccessException
+					| IllegalArgumentException | InvocationTargetException e) {
+				e.printStackTrace();
+				return null;
+			}
+		}
+
+		private void writeFailed() {
+			if (!hasDrainingStarted)
+				return;
+
+			if (drainBuffer != null)
+				throw new IllegalStateException("drainBuffer is full");
+
+			if (lastWrittenTime == 0) {
+				lastWrittenTime = System.nanoTime();
+				return;
+			}
+
+			if (System.nanoTime() - lastWrittenTime > gap) {
+				createDrainBuffer();
+			}
+		}
+	}
+}

From 82f417f733cce6cd8c6444ee0baff6cdc4ef5e4e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Sep 2014 13:10:09 +0800
Subject: [PATCH 362/881] drainingStarted() takes DrainType as arg.

Implementations may use the DrainType information for successful and deadlock
free draining.
---
 .../mit/streamjit/impl/distributed/node/LocalBuffer.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
index 316424aa..299a7f29 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
@@ -6,11 +6,12 @@
 import java.util.List;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.util.ConstructorSupplier;
 import edu.mit.streamjit.util.ReflectionUtils;
 
 /**
- * {@link LocalBuffer} connects up blob and down blob where both are running to
+ * {@link LocalBuffer} connects up blob and down blob where both are running at
  * same StreamNode. In this case, we can simply use any {@link Buffer}
  * implementation to connect up blob and down blob. But at the draining time
  * blobs write large amount of data, which blobs buffered inside during the init
@@ -23,7 +24,7 @@
  */
 public interface LocalBuffer extends Buffer {
 
-	public void drainingStarted();
+	public void drainingStarted(DrainType drainType);
 
 	/**
 	 * Modified version of {@link DynamicBufferManager#DynamicBuffer}. Instead
@@ -109,7 +110,7 @@ public int capacity() {
 		}
 
 		@Override
-		public void drainingStarted() {
+		public void drainingStarted(DrainType drainType) {
 			hasDrainingStarted = true;
 		}
 

From 8ada4915e44b2b99d43c840b4a8229e8425ad46c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Sep 2014 14:31:29 +0800
Subject: [PATCH 363/881] StreamNodes use LocalBuffers

---
 .../distributed/node/BlobsManagerImpl.java    | 17 ++++++++++---
 .../impl/distributed/node/BufferManager.java  | 24 +++++++++++++------
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 5cf535c4..2b4539ff 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -257,6 +257,8 @@ private class BlobExecuter {
 		 */
 		private ImmutableMap<Token, Buffer> bufferMap;
 
+		private ImmutableMap<Token, LocalBuffer> outputLocalBuffers;
+
 		/**
 		 * This flag will be set to true if an exception thrown by the core code
 		 * of the {@link Blob}. Any exception occurred in a blob's corecode will
@@ -321,7 +323,9 @@ public Token getBlobID() {
 		private ImmutableMap<Token, Buffer> buildBufferMap() {
 			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
 					.builder();
-			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
+			ImmutableMap.Builder<Token, LocalBuffer> outputLocalBufferBuilder = ImmutableMap
+					.builder();
+			ImmutableMap<Token, LocalBuffer> localBufferMap = bufferManager
 					.localBufferMap();
 			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
 					.inputChannelsMap();
@@ -344,7 +348,9 @@ private ImmutableMap<Token, Buffer> buildBufferMap() {
 			for (Token t : blob.getOutputs()) {
 				if (localBufferMap.containsKey(t)) {
 					assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
-					bufferMapBuilder.put(t, localBufferMap.get(t));
+					LocalBuffer buf = localBufferMap.get(t);
+					bufferMapBuilder.put(t, buf);
+					outputLocalBufferBuilder.put(t, buf);
 				} else if (outputChannels.containsKey(t)) {
 					BoundaryOutputChannel chnl = outputChannels.get(t);
 					bufferMapBuilder.put(t, chnl.getBuffer());
@@ -353,6 +359,7 @@ private ImmutableMap<Token, Buffer> buildBufferMap() {
 							"No Buffer for output channel %s ", t));
 				}
 			}
+			outputLocalBuffers = outputLocalBufferBuilder.build();
 			return bufferMapBuilder.build();
 		}
 
@@ -368,6 +375,10 @@ private void doDrain(DrainType drainType) {
 			if (drainType != DrainType.DISCARD)
 				inChnlManager.waitToStop();
 
+			for (LocalBuffer buf : outputLocalBuffers.values()) {
+				buf.drainingStarted(drainType);
+			}
+
 			if (this.blob != null) {
 				DrainCallback dcb = new DrainCallback(this);
 				drainState = 2;
@@ -847,7 +858,7 @@ private void clean(Buffer b, Token t) {
 		 * controller as a drain data.
 		 */
 		private boolean copyLocalBuffers() {
-			ImmutableMap<Token, Buffer> localBufferMap = bufferManager
+			ImmutableMap<Token, LocalBuffer> localBufferMap = bufferManager
 					.localBufferMap();
 			boolean areAllDrained = true;
 			for (BlobExecuter be : blobExecuters.values()) {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index 14212425..0696145c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -1,7 +1,9 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -11,8 +13,8 @@
 
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
+import edu.mit.streamjit.impl.distributed.node.LocalBuffer.LocalBuffer1;
 
 /**
  * {@link BlobsManager} will use the services from {@link BufferManager}.
@@ -62,7 +64,7 @@ public interface BufferManager {
 	 * @return local buffers if buffer sizes are calculated. Otherwise returns
 	 *         null.
 	 */
-	ImmutableMap<Token, Buffer> localBufferMap();
+	ImmutableMap<Token, LocalBuffer> localBufferMap();
 
 	public static abstract class AbstractBufferManager implements BufferManager {
 
@@ -78,7 +80,7 @@ public static abstract class AbstractBufferManager implements BufferManager {
 
 		protected ImmutableMap<Token, Integer> bufferSizes;
 
-		ImmutableMap<Token, Buffer> localBufferMap;
+		ImmutableMap<Token, LocalBuffer> localBufferMap;
 
 		public AbstractBufferManager(Set<Blob> blobSet) {
 			this.blobSet = blobSet;
@@ -128,19 +130,27 @@ public boolean isbufferSizesReady() {
 		}
 
 		@Override
-		public ImmutableMap<Token, Buffer> localBufferMap() {
+		public ImmutableMap<Token, LocalBuffer> localBufferMap() {
 			return localBufferMap;
 		}
 
 		protected final void createLocalBuffers() {
-			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-					.<Token, Buffer> builder();
+			ImmutableMap.Builder<Token, LocalBuffer> bufferMapBuilder = ImmutableMap
+					.<Token, LocalBuffer> builder();
 			for (Token t : localTokens) {
 				int bufSize = bufferSizes.get(t);
-				bufferMapBuilder.put(t, new ConcurrentArrayBuffer(bufSize));
+				bufferMapBuilder.put(t, concurrentArrayLocalBuffer(t, bufSize));
 			}
 			localBufferMap = bufferMapBuilder.build();
 		}
+
+		protected final LocalBuffer1 concurrentArrayLocalBuffer(Token t,
+				int bufSize) {
+			List<Object> args = new ArrayList<>(1);
+			args.add(bufSize);
+			return new LocalBuffer1(t.toString(), ConcurrentArrayBuffer.class,
+					args, bufSize, 0);
+		}
 	}
 
 	/**

From e6a34b99937c49677c6dff917d258686d3d80fbc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 29 Sep 2014 16:31:43 +0800
Subject: [PATCH 364/881] ConcurrentArrayLocalBuffer added.

---
 .../impl/distributed/node/LocalBuffer.java    | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
index 299a7f29..4fb93bbe 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
@@ -6,6 +6,7 @@
 import java.util.List;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.util.ConstructorSupplier;
 import edu.mit.streamjit.util.ReflectionUtils;
@@ -26,6 +27,62 @@ public interface LocalBuffer extends Buffer {
 
 	public void drainingStarted(DrainType drainType);
 
+	/**
+	 * Just a wrapper for {@link ConcurrentArrayBuffer}
+	 */
+	public class ConcurrentArrayLocalBuffer implements LocalBuffer {
+		private final ConcurrentArrayBuffer buffer;
+
+		public ConcurrentArrayLocalBuffer(int capacity) {
+			buffer = new ConcurrentArrayBuffer(capacity);
+		}
+
+		@Override
+		public Object read() {
+			return buffer.read();
+		}
+
+		@Override
+		public int read(Object[] data, int offset, int length) {
+			return buffer.read(data, offset, length);
+		}
+
+		@Override
+		public boolean readAll(Object[] data) {
+			return buffer.readAll(data);
+		}
+
+		@Override
+		public boolean readAll(Object[] data, int offset) {
+			return buffer.readAll(data, offset);
+		}
+
+		@Override
+		public boolean write(Object t) {
+			return buffer.write(t);
+		}
+
+		@Override
+		public int write(Object[] data, int offset, int length) {
+			return buffer.write(data, offset, length);
+		}
+
+		@Override
+		public int size() {
+			return buffer.size();
+		}
+
+		@Override
+		public int capacity() {
+			return buffer.capacity();
+		}
+
+		@Override
+		public void drainingStarted(DrainType drainType) {
+			System.out.println("drainingStarted: Not supported");
+		}
+	}
+
 	/**
 	 * Modified version of {@link DynamicBufferManager#DynamicBuffer}. Instead
 	 * of dynamically increase the buffer sizes, this implementation creates an

From 84479cc771f3fcdde99efbf379eace4f96fd8c51 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 29 Sep 2014 16:33:13 +0800
Subject: [PATCH 365/881] Method Renamed.

ConcurrentArrayLocalBuffer->ConcurrentArrayLocalBuffer1
---
 .../mit/streamjit/impl/distributed/node/BufferManager.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index 0696145c..61158cd5 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -139,12 +139,12 @@ protected final void createLocalBuffers() {
 					.<Token, LocalBuffer> builder();
 			for (Token t : localTokens) {
 				int bufSize = bufferSizes.get(t);
-				bufferMapBuilder.put(t, concurrentArrayLocalBuffer(t, bufSize));
+				bufferMapBuilder.put(t, concurrentArrayLocalBuffer1(t, bufSize));
 			}
 			localBufferMap = bufferMapBuilder.build();
 		}
 
-		protected final LocalBuffer1 concurrentArrayLocalBuffer(Token t,
+		protected final LocalBuffer1 concurrentArrayLocalBuffer1(Token t,
 				int bufSize) {
 			List<Object> args = new ArrayList<>(1);
 			args.add(bufSize);

From bdb58d482c07db284f536c8b783412a3acb746aa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 29 Sep 2014 16:36:55 +0800
Subject: [PATCH 366/881] ConcurrentArrayLocalBuffer() added.

This method does nothing but returns new ConcurrentArrayLocalBuffer object.
---
 .../streamjit/impl/distributed/node/BufferManager.java   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index 61158cd5..a6ff4c64 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -14,6 +14,7 @@
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
+import edu.mit.streamjit.impl.distributed.node.LocalBuffer.ConcurrentArrayLocalBuffer;
 import edu.mit.streamjit.impl.distributed.node.LocalBuffer.LocalBuffer1;
 
 /**
@@ -139,7 +140,8 @@ protected final void createLocalBuffers() {
 					.<Token, LocalBuffer> builder();
 			for (Token t : localTokens) {
 				int bufSize = bufferSizes.get(t);
-				bufferMapBuilder.put(t, concurrentArrayLocalBuffer1(t, bufSize));
+				bufferMapBuilder
+						.put(t, concurrentArrayLocalBuffer1(t, bufSize));
 			}
 			localBufferMap = bufferMapBuilder.build();
 		}
@@ -151,6 +153,11 @@ protected final LocalBuffer1 concurrentArrayLocalBuffer1(Token t,
 			return new LocalBuffer1(t.toString(), ConcurrentArrayBuffer.class,
 					args, bufSize, 0);
 		}
+
+		protected final LocalBuffer concurrentArrayLocalBuffer(Token t,
+				int bufSize) {
+			return new ConcurrentArrayLocalBuffer(bufSize);
+		}
 	}
 
 	/**

From 0b4f0ad719ca4cf92e98c547476cf4e9a6403902 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Sep 2014 11:35:13 +0800
Subject: [PATCH 367/881] Refactored : updateDrainDataStatistics() added.

AbstractDrainer.getDrainData() has been splitted and updateDrainDataStatistics()
added.
---
 .../impl/common/AbstractDrainer.java          | 40 ++++++++-----------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index a762ba82..13496b84 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -246,12 +246,10 @@ public final DrainData getDrainData() {
 				.builder();
 		for (Token t : Sets.union(boundaryInputData.keySet(),
 				boundaryOutputData.keySet())) {
-			ImmutableList<Object> in = boundaryInputData.get(t) != null
-					? boundaryInputData.get(t)
-					: ImmutableList.of();
-			ImmutableList<Object> out = boundaryOutputData.get(t) != null
-					? boundaryOutputData.get(t)
-					: ImmutableList.of();
+			ImmutableList<Object> in = boundaryInputData.get(t) != null ? boundaryInputData
+					.get(t) : ImmutableList.of();
+			ImmutableList<Object> out = boundaryOutputData.get(t) != null ? boundaryOutputData
+					.get(t) : ImmutableList.of();
 			dataBuilder.put(t, ImmutableList.builder().addAll(in).addAll(out)
 					.build());
 		}
@@ -259,7 +257,11 @@ public final DrainData getDrainData() {
 		ImmutableTable<Integer, String, Object> state = ImmutableTable.of();
 		DrainData draindata1 = new DrainData(dataBuilder.build(), state);
 		drainData = drainData.merge(draindata1);
+		updateDrainDataStatistics(drainData, false);
+		return drainData;
+	}
 
+	private void updateDrainDataStatistics(DrainData drainData, boolean print) {
 		if (drainDataStatistics == null) {
 			drainDataStatistics = new HashMap<>();
 			for (Token t : drainData.getData().keySet()) {
@@ -268,26 +270,16 @@ public final DrainData getDrainData() {
 		}
 
 		for (Token t : drainData.getData().keySet()) {
-			// System.out.print("Aggregated data: " + t.toString() + " - "
-			// + drainData.getData().get(t).size() + " - ");
-			// for (Object o : drainData.getData().get(t)) {
-			// System.out.print(o.toString() + ", ");
-			// }
-			// System.out.print('\n');
-
 			drainDataStatistics.get(t).add(drainData.getData().get(t).size());
+			if (print) {
+				System.out.print("Aggregated data: " + t.toString() + " - "
+						+ drainData.getData().get(t).size() + " - ");
+				// for (Object o : drainData.getData().get(t)) {
+				// System.out.print(o.toString() + ", ");
+				// }
+				System.out.print('\n');
+			}
 		}
-
-		// dumps the drain data into a file.
-		// try {
-		// FileOutputStream fout = new FileOutputStream("DrainData");
-		// ObjectOutputStream oos = new ObjectOutputStream(fout);
-		// oos.writeObject(drainData);
-		// oos.close();
-		// } catch (Exception ex) {
-		// ex.printStackTrace();
-		// }
-		return drainData;
 	}
 
 	/**

From 962639e2ef2be97519b07c2cb00940cc60702208 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Sep 2014 11:39:20 +0800
Subject: [PATCH 368/881] dumpDrainData() method added.

Just serializes DrainData into a file for debugging purpose.
---
 .../mit/streamjit/impl/common/AbstractDrainer.java | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 13496b84..b0d24871 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -2,8 +2,10 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -258,6 +260,7 @@ public final DrainData getDrainData() {
 		DrainData draindata1 = new DrainData(dataBuilder.build(), state);
 		drainData = drainData.merge(draindata1);
 		updateDrainDataStatistics(drainData, false);
+		// dumpDrainData(drainData);
 		return drainData;
 	}
 
@@ -282,6 +285,17 @@ private void updateDrainDataStatistics(DrainData drainData, boolean print) {
 		}
 	}
 
+	private void dumpDrainData(DrainData drainData) {
+		try {
+			FileOutputStream fout = new FileOutputStream("DrainData");
+			ObjectOutputStream oos = new ObjectOutputStream(fout);
+			oos.writeObject(drainData);
+			oos.close();
+		} catch (Exception ex) {
+			ex.printStackTrace();
+		}
+	}
+
 	/**
 	 * logs the size of the drain data on each channel for every draining and
 	 * writes down the statistics into a file.

From 871d66dc626400fba7a329e3de192bdf2309dafe Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Sep 2014 12:27:26 +0800
Subject: [PATCH 369/881] LocalBuffer1 reads and writes from drainBuffer.

---
 .../impl/distributed/node/LocalBuffer.java    | 83 +++++++++++++------
 1 file changed, 58 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
index 4fb93bbe..55dfb06c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/LocalBuffer.java
@@ -156,13 +156,26 @@ public LocalBuffer1(String name, Class<? extends Buffer> bufferClass,
 			this.writeBuffer = defaultBuffer;
 			this.gap = 10_000_000_000l; // 10s
 			hasDrainingStarted = false;
-			lastWrittenTime = 0;
 		}
 
+		/*
+		 * (non-Javadoc)
+		 * 
+		 * @see edu.mit.streamjit.impl.blob.Buffer#capacity()
+		 * 
+		 * if hasDrainingStarted == true sends defaultBuffer.capacity() * 2 as
+		 * capacity because if the room is 0 blobs never try to call write and
+		 * hence wirteFailed() will never be called. Check
+		 * Interpreter#pushOutputs().
+		 */
 		@Override
 		public int capacity() {
-			int cap = drainBuffer == null ? defaultBuffer.capacity()
-					: drainBuffer.capacity() + defaultBuffer.capacity();
+			int cap;
+			if (hasDrainingStarted) {
+				cap = drainBuffer == null ? defaultBuffer.capacity() * 2
+						: drainBuffer.capacity() + defaultBuffer.capacity();
+			} else
+				cap = defaultBuffer.capacity();
 			return cap;
 		}
 
@@ -173,25 +186,51 @@ public void drainingStarted(DrainType drainType) {
 
 		@Override
 		public Object read() {
-			Object o = defaultBuffer.read();
+			Object o;
+			if (drainBuffer == null || defaultBuffer.size() > 0)
+				o = defaultBuffer.read();
+			else
+				o = drainBuffer.read();
 			return o;
 		}
 
 		@Override
 		public int read(Object[] data, int offset, int length) {
-			int ret = defaultBuffer.read(data, offset, length);
+			int ret;
+			if (drainBuffer == null)
+				ret = defaultBuffer.read(data, offset, length);
+			else if (defaultBuffer.size() == 0)
+				ret = drainBuffer.read(data, offset, length);
+			else {
+				ret = defaultBuffer.read(data, offset, length);
+				ret += drainBuffer.read(data, offset + ret, length - ret);
+			}
 			return ret;
 		}
 
 		@Override
 		public boolean readAll(Object[] data) {
-			boolean ret = defaultBuffer.readAll(data);
-			return ret;
+			return readAll(data, 0);
 		}
 
 		@Override
 		public boolean readAll(Object[] data, int offset) {
-			boolean ret = defaultBuffer.readAll(data, offset);
+			boolean ret;
+			int need = data.length - offset;
+			if (drainBuffer == null)
+				ret = defaultBuffer.readAll(data, offset);
+			else if (defaultBuffer.size() == 0)
+				ret = drainBuffer.readAll(data, offset);
+			else if (need > defaultBuffer.size() + drainBuffer.size())
+				ret = false;
+			else {
+				int read = defaultBuffer.read(data, offset, need);
+				read += drainBuffer.read(data, offset + read, need - read);
+				ret = true;
+				if (read != need)
+					throw new IllegalStateException(
+							"data buffer is not full. Check the logic");
+			}
 			return ret;
 		}
 
@@ -207,31 +246,29 @@ public boolean write(Object t) {
 			boolean ret = writeBuffer.write(t);
 			if (!ret)
 				writeFailed();
-			else if (lastWrittenTime != 0)
-				lastWrittenTime = 0;
 			return ret;
 		}
 
 		@Override
 		public int write(Object[] data, int offset, int length) {
 			int written = writeBuffer.write(data, offset, length);
-			if (written == 0)
+			if (written != length) {
 				writeFailed();
-			else if (lastWrittenTime != 0)
-				lastWrittenTime = 0;
+				written += writeBuffer.write(data, offset + written, length
+						- written);
+			}
 			return written;
 		}
 
 		private void createDrainBuffer() {
 			assert drainBuffer == null : "drainBuffer has already been created.";
-			int newCapacity = 2 * defaultBuffer.capacity();
+			int newCapacity = 4 * defaultBuffer.capacity();
 			System.out
 					.println(String
 							.format("%s : Creating drain buffer: defaultBufferCapacity - %d, drainBufferCapacity - %d",
 									name, initialCapacity, newCapacity));
 			drainBuffer = getNewBuffer(newCapacity);
 			this.writeBuffer = drainBuffer;
-			lastWrittenTime = 0;
 		}
 
 		private List<?> getArguments(int newCapacity) {
@@ -262,16 +299,12 @@ private void writeFailed() {
 				return;
 
 			if (drainBuffer != null)
-				throw new IllegalStateException("drainBuffer is full");
-
-			if (lastWrittenTime == 0) {
-				lastWrittenTime = System.nanoTime();
-				return;
-			}
-
-			if (System.nanoTime() - lastWrittenTime > gap) {
-				createDrainBuffer();
-			}
+				throw new IllegalStateException(
+						String.format(
+								"drainBuffer is full "
+										+ "drainBuffer.size() = %d, defaultBuffer.size() = %d  ",
+								drainBuffer.size(), defaultBuffer.size()));
+			createDrainBuffer();
 		}
 	}
 }

From 1562d1a8ced130cce8e85a9161a535a9a9204c6a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 13 Oct 2014 17:11:53 +0800
Subject: [PATCH 370/881] Periodically prints no of outputs generated.

---
 .../impl/distributed/TailChannel.java         | 45 ++++++++++++++++++-
 .../distributed/common/GlobalConstants.java   |  6 ++-
 2 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 11d791a2..cb5cadf9 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -3,6 +3,8 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -31,6 +33,18 @@ public class TailChannel extends BlockingInputChannel {
 
 	private boolean skipLatchUp;
 
+	/**
+	 * Periodically prints no of outputs generated. See
+	 * {@link #printOutputCount()}.
+	 */
+	private ScheduledExecutorService scheduledExecutorService;
+
+	/**
+	 * The no of outputs generated at the end of last period. See
+	 * {@link #printOutputCount()}.
+	 */
+	private int lastCount;
+
 	/**
 	 * @param buffer
 	 * @param conProvider
@@ -61,6 +75,7 @@ public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 			pLogger = new PerformanceLogger();
 			pLogger.start();
 		}
+		printOutputCount();
 	}
 
 	@Override
@@ -68,8 +83,8 @@ public void receiveData() {
 		super.receiveData();
 		count++;
 
-		if (GlobalConstants.printOutputCount && count % 10000 == 0)
-			System.err.println(count);
+		// if (GlobalConstants.printOutputCount && count % 10000 == 0)
+		// System.err.println(count);
 
 		if (skipLatchUp && count > skipCount) {
 			skipLatch.countDown();
@@ -87,6 +102,8 @@ public void stop(int type) {
 			reset();
 			pLogger.stopLogging();
 		}
+		if (scheduledExecutorService != null)
+			scheduledExecutorService.shutdown();
 	}
 
 	/**
@@ -109,6 +126,30 @@ public long getFixedOutputTime() throws InterruptedException {
 		return normalizedTime;
 	}
 
+	/**
+	 * Periodically prints no of outputs generated.
+	 */
+	private void printOutputCount() {
+		if (GlobalConstants.printOutputCountPeriod < 1)
+			return;
+
+		lastCount = 0;
+		scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+		scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+
+			@Override
+			public void run() {
+				int newOutputs = count - lastCount;
+				lastCount = count;
+				System.out.println(String.format(
+						"Outputs: Total - %d, During last period - %d", count,
+						newOutputs));
+
+			}
+		}, GlobalConstants.printOutputCountPeriod,
+				GlobalConstants.printOutputCountPeriod, TimeUnit.MILLISECONDS);
+	}
+
 	public void reset() {
 		steadyLatch.countDown();
 		steadyLatch = new CountDownLatch(1);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 18b43aca..72225551 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -126,9 +126,11 @@ private GlobalConstants() {
 	public static final boolean useCompilerBlob = true;
 
 	/**
-	 * Prints number of outputs generated.
+	 * Period to print output count periodically. This printing feature get
+	 * turned off if this value is less than 1. Time unit is ms. See
+	 * {@link TailChannel}.
 	 */
-	public static final boolean printOutputCount = false;
+	public static final int printOutputCountPeriod = 6000;
 
 	/**
 	 * Enables {@link DistributedStreamCompiler} to run on a single node. When

From 3911d657a8e2496dff67bc0212b2f27e018b3d3e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 14 Oct 2014 07:32:34 +0800
Subject: [PATCH 371/881] Commented code removed.

---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index cb5cadf9..c838e6c5 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -83,9 +83,6 @@ public void receiveData() {
 		super.receiveData();
 		count++;
 
-		// if (GlobalConstants.printOutputCount && count % 10000 == 0)
-		// System.err.println(count);
-
 		if (skipLatchUp && count > skipCount) {
 			skipLatch.countDown();
 			skipLatchUp = false;

From 564912cd99486847c99bc2e504de8d7b4a64dcba Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 14 Oct 2014 12:55:37 +0800
Subject: [PATCH 372/881] Print message modified.

TailChannel's periodic out put count print message is modified.
---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index c838e6c5..cbd0248e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -139,7 +139,8 @@ public void run() {
 				int newOutputs = count - lastCount;
 				lastCount = count;
 				System.out.println(String.format(
-						"Outputs: Total - %d, During last period - %d", count,
+						"Outputs: since started - %d, during last %d ms - %d",
+						count, GlobalConstants.printOutputCountPeriod,
 						newOutputs));
 
 			}

From 9ea73e9f811c2ccea86e358d76cb73de67a9664d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 18 Oct 2014 09:52:54 +0800
Subject: [PATCH 373/881] FinalSend() method moved up

---
 .../node/BlockingOutputChannel.java           | 29 +++++++++----------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index f1f0b149..0bb17816 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -161,19 +161,6 @@ public final void sendData() {
 		}
 	}
 
-	@Override
-	public final void stop(boolean isFinal) {
-		if (debugLevel > 0)
-			System.out.println(Thread.currentThread().getName()
-					+ " - stop request");
-		if (!this.stopFlag.get()) {
-			this.isFinal = isFinal;
-			this.stopFlag.set(true);
-		} else if (debugLevel > 0) {
-			System.err.println("Stop has already been called.");
-		}
-	}
-
 	/**
 	 * This can be called when running the application with the final scheduling
 	 * configurations. Shouldn't be called when autotuner tunes.
@@ -206,14 +193,26 @@ private void finalSend() {
 		}
 	}
 
+	@Override
+	public final void stop(boolean isFinal) {
+		if (debugLevel > 0)
+			System.out.println(Thread.currentThread().getName()
+					+ " - stop request");
+		if (!this.stopFlag.get()) {
+			this.isFinal = isFinal;
+			this.stopFlag.set(true);
+		} else if (debugLevel > 0) {
+			System.err.println("Stop has already been called.");
+		}
+	}
+
 	private void reConnect() {
 		try {
 			this.connection.closeConnection();
 			while (!stopFlag.get()) {
 				System.out.println("TCPOutputChannel : Reconnecting...");
 				try {
-					this.connection = conProvider.getConnection(conInfo,
-							1000);
+					this.connection = conProvider.getConnection(conInfo, 1000);
 					return;
 				} catch (SocketTimeoutException stex) {
 					// We make this exception to recheck the stopFlag. Otherwise

From 64efc4bc3dde9e54e86a1b98ae3991fa225283b8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 18 Oct 2014 10:08:25 +0800
Subject: [PATCH 374/881] Refactored: new method send() added.

A new method send() added. sendData() and finalSend() use send() to send data.
---
 .../node/BlockingOutputChannel.java           | 75 +++++++------------
 1 file changed, 29 insertions(+), 46 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
index 0bb17816..020d2057 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingOutputChannel.java
@@ -135,29 +135,7 @@ public void run() {
 
 	public final void sendData() {
 		while (this.buffer.size() > 0 && !stopFlag.get()) {
-			try {
-				Object obj = buffer.read();
-				connection.writeObject(obj);
-				count++;
-
-				if (debugLevel == 3) {
-					System.out.println(Thread.currentThread().getName() + " - "
-							+ obj.toString());
-				}
-
-				if (writer != null) {
-					writer.write(obj.toString());
-					writer.write('\n');
-				}
-			} catch (IOException e) {
-				System.err
-						.println("TCP Output Channel. WriteObject exception.");
-				reConnect();
-			}
-			if (count % 1000 == 0 && debugLevel == 2) {
-				System.out.println(Thread.currentThread().getName() + " - "
-						+ count + " items have been sent");
-			}
+			send();
 		}
 	}
 
@@ -167,29 +145,7 @@ public final void sendData() {
 	 */
 	private void finalSend() {
 		while (this.buffer.size() > 0) {
-			try {
-				Object o = buffer.read();
-				connection.writeObject(o);
-				count++;
-
-				if (debugLevel == 3) {
-					System.out.println(Thread.currentThread().getName()
-							+ " FinalSend - " + o.toString());
-				}
-
-				if (writer != null) {
-					writer.write(o.toString());
-					writer.write('\n');
-				}
-
-			} catch (IOException e) {
-				System.err.println("TCP Output Channel. finalSend exception.");
-			}
-			if (count % 1000 == 0 && debugLevel == 2) {
-				System.out.println(Thread.currentThread().getName()
-						+ " FinalSend - " + count
-						+ " no of items have been sent");
-			}
+			send();
 		}
 	}
 
@@ -224,6 +180,33 @@ private void reConnect() {
 		}
 	}
 
+	private void send() {
+		try {
+			Object o = buffer.read();
+			connection.writeObject(o);
+			count++;
+
+			if (debugLevel == 3) {
+				System.out.println(Thread.currentThread().getName()
+						+ " Send - " + o.toString());
+			}
+
+			if (writer != null) {
+				writer.write(o.toString());
+				writer.write('\n');
+			}
+
+		} catch (IOException e) {
+			e.printStackTrace();
+			System.err.println("TCP Output Channel. Send exception.");
+			reConnect();
+		}
+		if (count % 1000 == 0 && debugLevel == 2) {
+			System.out.println(Thread.currentThread().getName() + " Send - "
+					+ count + " no of items have been sent");
+		}
+	}
+
 	@Override
 	public final String name() {
 		return name;

From 3869511db3c29b8a6ed213f017fbd4e9ed020e9e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 18 Oct 2014 14:24:12 +0800
Subject: [PATCH 375/881] Comments added to DrainType's enum elements.

---
 .../distributed/common/CTRLRDrainElement.java | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java b/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
index 893518e5..4864dfba 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/CTRLRDrainElement.java
@@ -96,22 +96,24 @@ public interface CTRLRDrainProcessor {
 
 	/**
 	 * Three types of draining are possible.
-	 * <ol>
-	 * <li>drainType = 1 : Final draining. No drain data. All {@link Blob}s are
-	 * expected to run and finish all data.
-	 * <li>drainType = 2 : Intermediate draining. Drain data is required in this
-	 * mode. {@link BoundaryInputChannel}s may create extra buffer and put all
-	 * unconsumed data, and finally send this drain data to the
-	 * {@link Controller} for reconfiguration.
-	 * <li>drainType = 3 : Discard all unconsumed data. This is useful, if we
-	 * don't care about the data while tuning for performance.
-	 * </ol>
-	 * 
-	 * See the same description at {@link BoundaryInputChannel#stop(int)}.
-	 * 
 	 */
 	public enum DrainType {
-		FINAL(1), INTERMEDIATE(2), DISCARD(3);
+		/**
+		 * Final draining. No drain data. All {@link Blob}s are expected to run
+		 * and finish data in input buffers buffers.
+		 */
+		FINAL(1), /**
+		 * Intermediate draining. Drain data is required in this mode.
+		 * {@link BoundaryInputChannel}s may create extra buffer and put all
+		 * unconsumed data, and finally send this drain data to the
+		 * {@link Controller} for reconfiguration.
+		 */
+		INTERMEDIATE(2), /**
+		 * Discard all unconsumed data. This is useful, if we
+		 * don't care about the data while tuning for performance.
+		 * 
+		 */
+		DISCARD(3);
 		private final int code;
 
 		DrainType(int code) {

From 3a7bc89be8dab440d66f64b4cc29b54b233ee999 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 19 Oct 2014 07:36:38 +0800
Subject: [PATCH 376/881] Methods decides based on useDrainData flag.

awaitDrainData() and getDrainData() methods changes their functionalities
based on GlobalConstants.useDrainData flag.
---
 src/edu/mit/streamjit/impl/common/AbstractDrainer.java   | 5 ++++-
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java | 9 +++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index b0d24871..09bb09e9 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -213,7 +213,8 @@ public final void drained(Token blobID) {
 	}
 
 	public final void awaitDrainData() throws InterruptedException {
-		drainDataLatch.await();
+		if (GlobalConstants.useDrainData)
+			drainDataLatch.await();
 	}
 
 	public final void newDrainData(DrainedData drainedData) {
@@ -231,6 +232,8 @@ public final void newDrainData(DrainedData drainedData) {
 	 * @return Aggregated DrainData after the draining.
 	 */
 	public final DrainData getDrainData() {
+		if (!GlobalConstants.useDrainData)
+			return null;
 		DrainData drainData = null;
 		Map<Token, ImmutableList<Object>> boundaryInputData = new HashMap<>();
 		Map<Token, ImmutableList<Object>> boundaryOutputData = new HashMap<>();
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 7f1d347a..8170656f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -164,12 +164,9 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				System.err.println("awaitDrainedIntrmdiate");
 				drainer.awaitDrainedIntrmdiate();
 
-				if (GlobalConstants.useDrainData) {
-					System.err.println("awaitDrainData...");
-					drainer.awaitDrainData();
-					DrainData drainData = drainer.getDrainData();
-					app.drainData = drainData;
-				}
+				drainer.awaitDrainData();
+				DrainData drainData = drainer.getDrainData();
+				app.drainData = drainData;
 			}
 
 			int multiplier = 1000;

From bffc6e891a5ee26bd77fba2921f87b6d8cabe611 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 19 Oct 2014 08:34:19 +0800
Subject: [PATCH 377/881] BoundaryInputChannel.stop(DrainType).

Method stop() takes DrainType as parameter. Previously it took int.
---
 .../impl/distributed/StreamJitAppManager.java       | 12 +++++++-----
 .../mit/streamjit/impl/distributed/TailChannel.java |  3 ++-
 .../impl/distributed/common/BoundaryChannel.java    | 13 ++-----------
 .../distributed/common/BoundaryChannelManager.java  |  5 +++--
 .../impl/distributed/node/BlobsManagerImpl.java     |  4 ++--
 .../impl/distributed/node/BlockingInputChannel.java |  6 +++---
 6 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 00f7a9ee..6a8b3818 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -141,12 +141,14 @@ public void drainingFinished(boolean isFinal) {
 		}
 
 		if (tailChannel != null) {
-			if (isFinal)
-				tailChannel.stop(1);
-			else if (GlobalConstants.useDrainData)
-				tailChannel.stop(2);
+			if (GlobalConstants.useDrainData)
+				if (isFinal)
+					tailChannel.stop(DrainType.FINAL);
+				else
+					tailChannel.stop(DrainType.INTERMEDIATE);
 			else
-				tailChannel.stop(3);
+				tailChannel.stop(DrainType.DISCARD);
+
 			try {
 				tailThread.join();
 			} catch (InterruptedException e) {
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index cbd0248e..4a7198db 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -12,6 +12,7 @@
 
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
@@ -93,7 +94,7 @@ public void receiveData() {
 	}
 
 	@Override
-	public void stop(int type) {
+	public void stop(DrainType type) {
 		super.stop(type);
 		if (pLogger != null) {
 			reset();
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
index cf1660c6..7f673201 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannel.java
@@ -3,6 +3,7 @@
 import com.google.common.collect.ImmutableList;
 
 import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 
 /**
@@ -55,20 +56,10 @@ public interface BoundaryInputChannel extends BoundaryChannel {
 		 * <p>
 		 * Based on the type argument, implementation may treat uncounsumed data
 		 * differently
-		 * <ol>
-		 * <li>1 - No extraBuffer. Wait and push all received data in to the
-		 * actual buffer. May be used at final draining.
-		 * <li>2 - Create extra buffer and put all unconsumed data. This can be
-		 * send to the controller as draindata. May be used at intermediate
-		 * draining.
-		 * <li>3 - Discard all unconsumed data. This is useful, if we don't care
-		 * about the data while tuning for performance.
 		 * </p>
 		 * 
-		 * @param type
-		 *            : Can be 1, 2 or 3. rest are illegal.
 		 */
-		void stop(int type);
+		void stop(DrainType type);
 
 		/**
 		 * Receive data from other node.
diff --git a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
index 8e0a2993..951b77cb 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/BoundaryChannelManager.java
@@ -10,6 +10,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
 
 /**
@@ -40,7 +41,7 @@ public interface BoundaryInputChannelManager extends BoundaryChannelManager {
 		 * @param stopType
 		 *            See {@link BoundaryInputChannel#stop(int)}
 		 */
-		void stop(int stopType);
+		void stop(DrainType stopType);
 	}
 
 	public interface BoundaryOutputChannelManager
@@ -101,7 +102,7 @@ public void waitToStop() {
 		}
 
 		@Override
-		public void stop(int stopType) {
+		public void stop(DrainType stopType) {
 			for (BoundaryInputChannel bc : inputChannels.values()) {
 				bc.stop(stopType);
 			}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2b4539ff..f93bd969 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -368,7 +368,7 @@ private void doDrain(DrainType drainType) {
 			this.drainType = drainType;
 			drainState = 1;
 
-			inChnlManager.stop(drainType.toint());
+			inChnlManager.stop(drainType);
 			// TODO: [2014-03-14] I commented following line to avoid one dead
 			// lock case when draining. Deadlock 5 and 6.
 			// [2014-09-17] Lets waitToStop() if drain data is required.
@@ -600,7 +600,7 @@ private void startChannels() {
 		}
 
 		private void stop() {
-			inChnlManager.stop(1);
+			inChnlManager.stop(DrainType.FINAL);
 			outChnlManager.stop(true);
 
 			for (Thread t : blobThreads) {
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 6af29a06..9eaaf711 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
@@ -345,10 +346,9 @@ private void reConnect() {
 	}
 
 	@Override
-	public void stop(int type) {
-		assert 0 < type && type < 4 : "Undefined stop type";
+	public void stop(DrainType type) {
 		if (this.stopType.get() == 0) {
-			stopType.set(type);
+			stopType.set(type.toint());
 		} else if (debugLevel > 0) {
 			System.err.println("Stop has already been called.");
 		}

From e174e1eb03c2ebd22f00e33f0da69a98bcfdf230 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 21 Oct 2014 05:16:18 +0800
Subject: [PATCH 378/881] Tag TODO added.

---
 .../streamjit/impl/distributed/runtimer/DistributedDrainer.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
index 86fab2a5..7c9da073 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
@@ -15,6 +15,7 @@ public class DistributedDrainer extends AbstractDrainer {
 
 	public DistributedDrainer(StreamJitAppManager manager) {
 		this.manager = manager;
+		//TODO:
 		// Read this. Don't let the "this" reference escape during construction
 		// http://www.ibm.com/developerworks/java/library/j-jtp0618/
 		manager.setDrainer(this);

From fb625919d16046791369bd96998c1edc9553f4e0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 21 Oct 2014 14:45:48 +0800
Subject: [PATCH 379/881] Configuration Analyzer has been added.

---
 .../impl/common/ConfigurationAnalyzer.java    | 167 ++++++++++++++++++
 1 file changed, 167 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
new file mode 100644
index 00000000..f9de6109
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -0,0 +1,167 @@
+package edu.mit.streamjit.impl.common;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map.Entry;
+
+import edu.mit.streamjit.impl.common.Configuration.FloatParameter;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
+import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+
+public class ConfigurationAnalyzer {
+
+	public static void main(String[] args) {
+		ConfigurationAnalyzer ca = new ConfigurationAnalyzer(
+				"NestedSplitJoinCore");
+		ca.compare(3, 4);
+	}
+
+	String appDir;
+
+	String appName;
+
+	List<Integer> bestConfigurations;
+
+	/**
+	 * Path of the directory which contains app's configuration in sub
+	 * directory.
+	 * 
+	 * <pre>
+	 * confgDirectory
+	 * 			|
+	 * 			------>appName1
+	 * 			|
+	 * 			------>appName2
+	 * 			|
+	 * 			------>
+	 * </pre>
+	 */
+
+	private final String cfgDirectory = "configurations";
+
+	public ConfigurationAnalyzer(String appName) {
+		verifyPath(cfgDirectory, appName);
+		bestConfigurations = new LinkedList<>();
+		this.appName = appName;
+		this.appDir = String.format("%s%s%s", cfgDirectory, File.separator,
+				appName);
+	}
+
+	private void compare(FloatParameter p1, FloatParameter p2) {
+		float val1 = p1.getValue();
+		float val2 = p2.getValue();
+		if (val1 == val2)
+			System.out.println(String.format("%s: p1 = p2. value = %f",
+					p1.getName(), val1));
+		if (val1 > val2)
+			System.out.println(String.format("%s: p1 > p2. %f > %f",
+					p1.getName(), val1, val2));
+		else
+			System.out.println(String.format("%s: p1 < p2. %f < %f",
+					p1.getName(), val1, val2));
+	}
+
+	private void compare(Integer first, Integer second) {
+		Configuration cfg1 = readcoConfiguration(appDir, appName, first);
+		Configuration cfg2 = readcoConfiguration(appDir, appName, second);
+		for (Entry<String, Parameter> en : cfg1.getParametersMap().entrySet()) {
+			Parameter p1 = en.getValue();
+			Parameter p2 = cfg2.getParameter(en.getKey());
+			if (p2 == null)
+				throw new IllegalStateException(String.format(
+						"No parameter %s in configuration2", en.getKey()));
+			if (p1.getClass() == Configuration.IntParameter.class)
+				compare((IntParameter) p1, (IntParameter) p2);
+			else if (p1.getClass() == Configuration.FloatParameter.class)
+				compare((FloatParameter) p1, (FloatParameter) p2);
+			else if (p1.getClass() == Configuration.SwitchParameter.class)
+				compare((SwitchParameter<?>) p1, (SwitchParameter<?>) p2);
+			else
+				System.out.println(String.format(
+						"Parameter class %s is not handled.", p1.getClass()
+								.getName()));
+
+		}
+	}
+
+	/*
+	 * Any way to avoid code duplication in compare(IntParameter p1,
+	 * IntParameter p2) and compare(FloatParameter p1, FloatParameter p2)?
+	 */
+	/**
+	 * 
+	 * @param p1
+	 * @param p2
+	 */
+	private void compare(IntParameter p1, IntParameter p2) {
+		int val1 = p1.getValue();
+		int val2 = p2.getValue();
+		if (val1 == val2)
+			System.out.println(String.format("%s: p1 = p2. value = %d",
+					p1.getName(), val1));
+		if (val1 > val2)
+			System.out.println(String.format("%s: p1 > p2. %d > %d",
+					p1.getName(), val1, val2));
+		else
+			System.out.println(String.format("%s: p1 < p2. %d < %d",
+					p1.getName(), val1, val2));
+	}
+
+	private <T1, T2> void compare(SwitchParameter<T1> p1, SwitchParameter<T2> p2) {
+		Class<T1> type1 = p1.getGenericParameter();
+		Class<T2> type2 = p2.getGenericParameter();
+		assert type1 == type2;
+		T1 val1 = p1.getValue();
+		T2 val2 = p2.getValue();
+		System.out.println(p1.getUniverse());
+		if (val1.equals(val2))
+			System.out.println("Same value");
+		else
+			System.out.println(String.format(
+					"Different values p1 = %s, p2 = %s", val1, val2));
+	}
+
+	private float getRunningTime() {
+		return 1.0f;
+	}
+
+	private Configuration readcoConfiguration(String appDir, String appName,
+			Integer cfgNo) {
+		String cfg = String.format("%s%s%d%s.cfg", appDir, File.separator,
+				cfgNo, appName);
+		return readConfiguration(cfg);
+	}
+
+	private Configuration readConfiguration(String name) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(name));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File reader error. No %s configuration file.", name));
+		}
+		return null;
+	}
+
+	private boolean verifyPath(String cfgDir, String appName) {
+		String dbPath = appName;
+		File db = new File(dbPath);
+		if (!db.exists())
+			throw new IllegalStateException("No database file found in "
+					+ dbPath);
+
+		String dirPath = String.format("%s%s%s", cfgDir, File.separator,
+				appName);
+		File dir = new File(dirPath);
+		if (!dir.exists())
+			throw new IllegalStateException("No directory found in " + dirPath);
+
+		return true;
+	}
+}

From b57c687453f16a84866e140be664c375395d81e1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 21 Oct 2014 17:46:32 +0800
Subject: [PATCH 380/881] Print message format changed to compareSwitch()

---
 .../mit/streamjit/impl/common/ConfigurationAnalyzer.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index f9de6109..ccb17348 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -117,12 +117,15 @@ private <T1, T2> void compare(SwitchParameter<T1> p1, SwitchParameter<T2> p2) {
 		assert type1 == type2;
 		T1 val1 = p1.getValue();
 		T2 val2 = p2.getValue();
-		System.out.println(p1.getUniverse());
+
 		if (val1.equals(val2))
-			System.out.println("Same value");
+			System.out.println(String.format(
+					"%s - same values - p1 = %s, p2 = %s. Universe:%s",
+					p1.getName(), val1, val2, p1.getUniverse()));
 		else
 			System.out.println(String.format(
-					"Different values p1 = %s, p2 = %s", val1, val2));
+					"%s - different values - p1 = %s, p2 = %s. Universe:%s",
+					p1.getName(), val1, val2, p1.getUniverse()));
 	}
 
 	private float getRunningTime() {

From 58cfc07d01542997cfe71076cdabb248a4f06885 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 24 Oct 2014 00:56:07 +0800
Subject: [PATCH 381/881] Inner class SqliteAdapter moved to a new file.

As ConfigGenerator.SqliteAdapter class is getting used by several other
classes, it has been moved to a new file.
---
 .../mit/streamjit/tuner/ConfigGenerator.java  | 81 +------------------
 src/edu/mit/streamjit/tuner/RunApp.java       |  5 +-
 .../mit/streamjit/tuner/SqliteAdapter.java    | 81 +++++++++++++++++++
 3 files changed, 85 insertions(+), 82 deletions(-)
 create mode 100644 src/edu/mit/streamjit/tuner/SqliteAdapter.java

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 1d5d1401..99654759 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -4,12 +4,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
 
 import com.google.common.collect.ImmutableSet;
 
@@ -49,9 +43,9 @@ public class ConfigGenerator {
 	public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		checkNotNull(provider);
 
-		sqliteAdapter sqlite;
+		SqliteAdapter sqlite;
 		try {
-			sqlite = new sqliteAdapter();
+			sqlite = new SqliteAdapter();
 		} catch (ClassNotFoundException e) {
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
@@ -106,77 +100,6 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		// new ProcessBuilder("xterm", "-e", "python", tunerPath).start();
 	}
 
-	public static class sqliteAdapter {
-
-		private Statement statement;
-		private Connection con = null;
-
-		public sqliteAdapter() throws ClassNotFoundException {
-			Class.forName("org.sqlite.JDBC");
-		}
-
-		public void connectDB(String path) {
-			try {
-				con = DriverManager.getConnection(String.format(
-						"jdbc:sqlite:%s", path));
-				statement = con.createStatement();
-				statement.setQueryTimeout(30); // set timeout to 30 sec.
-
-			} catch (SQLException e) {
-				e.printStackTrace();
-			}
-		}
-
-		/**
-		 * Creates table iff it is not exists
-		 * 
-		 * @param table
-		 *            Name of the table
-		 * @param signature
-		 *            Column format of the table
-		 * @throws SQLException
-		 */
-		public void createTable(String table, String signature) {
-			checkNotNull(con);
-			DatabaseMetaData dbm;
-			try {
-				dbm = con.getMetaData();
-
-				ResultSet tables = dbm.getTables(null, null, table, null);
-				if (!tables.next()) {
-					// "create table %s ()"
-					statement.executeUpdate(String.format(
-							"create table %s (%s)", table, signature));
-				}
-			} catch (SQLException e) {
-				// TODO Auto-generated catch block
-				e.printStackTrace();
-			}
-		}
-
-		public ResultSet executeQuery(String sql) {
-			try {
-				ResultSet rs = statement.executeQuery(sql);
-				// con.commit();
-				return rs;
-			} catch (SQLException e) {
-				e.printStackTrace();
-			}
-			return null;
-		}
-
-		public int executeUpdate(String sql) {
-			try {
-				int ret = statement.executeUpdate(sql);
-				// con.commit();
-				return ret;
-			} catch (SQLException e) {
-				e.printStackTrace();
-			}
-			return -1;
-		}
-	}
-
 	/**
 	 * @param args
 	 *            [0] - NoofMachines that will be connected in distributed case.
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index e0ca0db6..8d312971 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -13,7 +13,6 @@
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
-import edu.mit.streamjit.tuner.ConfigGenerator.sqliteAdapter;
 
 /**
  * {@link RunApp} reads configuration, streamJit's app name and location
@@ -44,9 +43,9 @@ public static void main(String[] args) throws SQLException {
 				benchmarkName, round));
 
 		String sjDbPath = "sj" + benchmarkName + ".db";
-		sqliteAdapter sjDb;
+		SqliteAdapter sjDb;
 		try {
-			sjDb = new sqliteAdapter();
+			sjDb = new SqliteAdapter();
 		} catch (ClassNotFoundException e1) {
 			System.err
 					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
diff --git a/src/edu/mit/streamjit/tuner/SqliteAdapter.java b/src/edu/mit/streamjit/tuner/SqliteAdapter.java
new file mode 100644
index 00000000..0789caaa
--- /dev/null
+++ b/src/edu/mit/streamjit/tuner/SqliteAdapter.java
@@ -0,0 +1,81 @@
+package edu.mit.streamjit.tuner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public class SqliteAdapter {
+
+	private Connection con = null;
+	private Statement statement;
+
+	public SqliteAdapter() throws ClassNotFoundException {
+		Class.forName("org.sqlite.JDBC");
+	}
+
+	public void connectDB(String path) {
+		try {
+			con = DriverManager.getConnection(String.format("jdbc:sqlite:%s",
+					path));
+			statement = con.createStatement();
+			statement.setQueryTimeout(30); // set timeout to 30 sec.
+
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}
+	}
+
+	/**
+	 * Creates table iff it is not exists
+	 * 
+	 * @param table
+	 *            Name of the table
+	 * @param signature
+	 *            Column format of the table
+	 * @throws SQLException
+	 */
+	public void createTable(String table, String signature) {
+		checkNotNull(con);
+		DatabaseMetaData dbm;
+		try {
+			dbm = con.getMetaData();
+
+			ResultSet tables = dbm.getTables(null, null, table, null);
+			if (!tables.next()) {
+				// "create table %s ()"
+				statement.executeUpdate(String.format("create table %s (%s)",
+						table, signature));
+			}
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
+	public ResultSet executeQuery(String sql) {
+		try {
+			ResultSet rs = statement.executeQuery(sql);
+			// con.commit();
+			return rs;
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}
+		return null;
+	}
+
+	public int executeUpdate(String sql) {
+		try {
+			int ret = statement.executeUpdate(sql);
+			// con.commit();
+			return ret;
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}
+		return -1;
+	}
+}
\ No newline at end of file

From 263709d6e92d087bb4191f3a2ba46910ad8c19a6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 24 Oct 2014 01:09:07 +0800
Subject: [PATCH 382/881] Constructor handles ClassNotFoundException.

Constructor of SqliteAdapter checks if org.sqlite.JDBC class exists.
Throws IllegalStateException if sqlite does not exist.
---
 src/edu/mit/streamjit/tuner/ConfigGenerator.java | 10 +---------
 src/edu/mit/streamjit/tuner/RunApp.java          | 10 +---------
 src/edu/mit/streamjit/tuner/SqliteAdapter.java   | 13 +++++++++++--
 3 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/ConfigGenerator.java b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
index 99654759..7ba3864f 100644
--- a/src/edu/mit/streamjit/tuner/ConfigGenerator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigGenerator.java
@@ -44,15 +44,7 @@ public void generate(BenchmarkProvider provider, BlobFactory factory) {
 		checkNotNull(provider);
 
 		SqliteAdapter sqlite;
-		try {
-			sqlite = new SqliteAdapter();
-		} catch (ClassNotFoundException e) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e.printStackTrace();
-			return;
-		}
-
+		sqlite = new SqliteAdapter();
 		String dbPath = "streamjit.db";
 		sqlite.connectDB(dbPath);
 		sqlite.createTable(
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index 8d312971..f0a0829b 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -44,15 +44,7 @@ public static void main(String[] args) throws SQLException {
 
 		String sjDbPath = "sj" + benchmarkName + ".db";
 		SqliteAdapter sjDb;
-		try {
-			sjDb = new SqliteAdapter();
-		} catch (ClassNotFoundException e1) {
-			System.err
-					.println("Sqlite3 database not found...couldn't update the database with the configutaion.");
-			e1.printStackTrace();
-			return;
-		}
-
+		sjDb = new SqliteAdapter();
 		sjDb.connectDB(sjDbPath);
 		ResultSet result = sjDb.executeQuery(String.format(
 				"SELECT * FROM results WHERE Round=%d", round));
diff --git a/src/edu/mit/streamjit/tuner/SqliteAdapter.java b/src/edu/mit/streamjit/tuner/SqliteAdapter.java
index 0789caaa..796ec9ec 100644
--- a/src/edu/mit/streamjit/tuner/SqliteAdapter.java
+++ b/src/edu/mit/streamjit/tuner/SqliteAdapter.java
@@ -14,8 +14,17 @@ public class SqliteAdapter {
 	private Connection con = null;
 	private Statement statement;
 
-	public SqliteAdapter() throws ClassNotFoundException {
-		Class.forName("org.sqlite.JDBC");
+	/**
+	 * Checks if org.sqlite.JDBC class exists. Throws
+	 * {@link IllegalStateException} if sqlite does not exist.
+	 */
+	public SqliteAdapter() {
+		try {
+			Class.forName("org.sqlite.JDBC");
+		} catch (ClassNotFoundException e) {
+			e.printStackTrace();
+			throw new IllegalStateException("Sqlite3 database not found.");
+		}
 	}
 
 	public void connectDB(String path) {

From 19ab808d7394e98cacf473e86c1b44e906df7589 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 24 Oct 2014 14:01:57 +0800
Subject: [PATCH 383/881] Method getRunningTime() has been added

Get running time of a round from opentuner's program database.
---
 .../impl/common/ConfigurationAnalyzer.java    | 27 ++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index ccb17348..73f2d2a9 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -3,6 +3,8 @@
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
@@ -11,13 +13,16 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.tuner.SqliteAdapter;
 
 public class ConfigurationAnalyzer {
 
 	public static void main(String[] args) {
 		ConfigurationAnalyzer ca = new ConfigurationAnalyzer(
 				"NestedSplitJoinCore");
-		ca.compare(3, 4);
+		// ca.compare(3, 4);
+
+		System.out.println(ca.getRunningTime("NestedSplitJoinCore", 3));
 	}
 
 	String appDir;
@@ -128,8 +133,24 @@ private <T1, T2> void compare(SwitchParameter<T1> p1, SwitchParameter<T2> p2) {
 					p1.getName(), val1, val2, p1.getUniverse()));
 	}
 
-	private float getRunningTime() {
-		return 1.0f;
+	private SqliteAdapter connectDB(String appName) {
+		SqliteAdapter sqlite = new SqliteAdapter();
+		sqlite.connectDB(appName);
+		return sqlite;
+	}
+
+	private double getRunningTime(String appName, int round) {
+		SqliteAdapter sqlite = connectDB(appName);
+		ResultSet result = sqlite.executeQuery(String.format(
+				"SELECT * FROM result WHERE id=%d", round));
+
+		String runtime = "1000000000";
+		try {
+			runtime = result.getString("time");
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}
+		return Double.parseDouble(runtime);
 	}
 
 	private Configuration readcoConfiguration(String appDir, String appName,

From 7ffeae502aa8a126409211945a095730d6875467 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 24 Oct 2014 14:10:41 +0800
Subject: [PATCH 384/881] Prints number of parameters exist in a cfg.

Just prints parameter count.
---
 .../impl/common/ConfigurationEditor.java      | 41 +++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index fc9c27c6..6bee6016 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -2,8 +2,10 @@
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileReader;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
@@ -41,7 +43,9 @@ public static void main(String[] args) throws IOException {
 		// edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
-		changeMultiplierVal();
+		// changeMultiplierVal();
+		// printCfgValues("1NestedSplitJoinCore.cfg");
+		 printAll("../Tuner layer/tuning-oopsla2014");
 	}
 
 	/**
@@ -346,7 +350,8 @@ private static String getConfigurationString(Configuration cfg) {
 	/**
 	 * Save the configuration.
 	 */
-	private static void saveConfg(Configuration config, int round, String appName) {
+	private static void saveConfg(Configuration config, int round,
+			String appName) {
 		String json = config.toJson();
 		try {
 
@@ -368,8 +373,23 @@ private static void saveConfg(Configuration config, int round, String appName) {
 		}
 	}
 
+	private static void printCfgValues(String fileName) {
+		Configuration cfg = readConfiguration(fileName);
+		File f = new File(fileName);
+		Map<String, Parameter> parameters = cfg.getParametersMap();
+		System.out.println(String.format("%s - %d", f.getName(), parameters
+				.entrySet().size()));
+		// int i = 0;
+		//
+		// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
+		// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
+		// }
+
+	}
+
 	private static Configuration readConfiguration(String simpeName) {
-		String name = String.format("%s.cfg", simpeName);
+		// String name = String.format("%s.cfg", simpeName);
+		String name = simpeName;
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(name));
 			String json = reader.readLine();
@@ -381,4 +401,19 @@ private static Configuration readConfiguration(String simpeName) {
 		}
 		return null;
 	}
+
+	private static void printAll(String folderPath) {
+		File folder = new File(folderPath);
+		File[] listOfFiles = folder.listFiles(new FilenameFilter() {
+
+			@Override
+			public boolean accept(File dir, String name) {
+				return name.toLowerCase().endsWith(".cfg");
+			}
+		});
+		System.out.println(String.format("Parameters in configuration..."));
+		for (int i = 0; i < listOfFiles.length; i++) {
+			printCfgValues(listOfFiles[i].getAbsolutePath());
+		}
+	}
 }

From 4e47253920728e89908b0e1abd466ba800a843b8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 01:14:53 +0800
Subject: [PATCH 385/881] ConfigurationUtils.java has been added.

ConfigurationUtils contains common utility methods those deal with Configuration.
---
 .../streamjit/util/ConfigurationUtils.java    | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 src/edu/mit/streamjit/util/ConfigurationUtils.java

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
new file mode 100644
index 00000000..9970a615
--- /dev/null
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -0,0 +1,42 @@
+package edu.mit.streamjit.util;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+
+import edu.mit.streamjit.impl.common.Configuration;
+
+/**
+ * {@link ConfigurationUtils} contains common utility methods those deal with
+ * {@link Configuration}.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since May 10, 2013
+ *
+ */
+public class ConfigurationUtils {
+
+	/**
+	 * @param cfgFilePath
+	 *            path of the configuration file that need to be read.
+	 * @return {@link Configuration} object if valid file exists. Otherwise
+	 *         returns null.
+	 */
+	public static Configuration readConfiguration(String cfgFilePath) {
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					cfgFilePath));
+			String json = reader.readLine();
+			reader.close();
+			return Configuration.fromJson(json);
+		} catch (IOException ex) {
+			System.err.println(String
+					.format("File reader error. No %s configuration file.",
+							cfgFilePath));
+		} catch (Exception ex) {
+			System.err.println(String.format(
+					"File %s is not a configuration file.", cfgFilePath));
+		}
+		return null;
+	}
+}

From ae9c9ccc7fd34dc8b488c6593f370a314f24cead Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 01:33:03 +0800
Subject: [PATCH 386/881] ConfigurationEditor uses ConfigurationUtils

ConfigurationEditor uses ConfigurationUtils.readConfiguration()
---
 .../impl/common/ConfigurationEditor.java      | 25 ++++++-------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 6bee6016..ab06c6eb 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -27,6 +27,7 @@
 import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
 import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
+import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 public class ConfigurationEditor {
@@ -45,14 +46,16 @@ public static void main(String[] args) throws IOException {
 		// convert();
 		// changeMultiplierVal();
 		// printCfgValues("1NestedSplitJoinCore.cfg");
-		 printAll("../Tuner layer/tuning-oopsla2014");
+		printAll("../Tuner layer/tuning-oopsla2014");
 	}
 
 	/**
 	 * Reads a configuration and changes its multiplier value.
 	 */
 	private static void changeMultiplierVal() {
-		Configuration config = readConfiguration("final_NestedSplitJoinCore");
+		String appName = "final_NestedSplitJoinCore";
+		Configuration config = ConfigurationUtils.readConfiguration(String
+				.format("%s.cfg", appName));
 		Configuration.Builder builder = Configuration.builder(config);
 		IntParameter mulParam = config.getParameter("multiplier",
 				IntParameter.class);
@@ -266,7 +269,8 @@ private static void print(String name) {
 
 	private static void convert() {
 		String appName = "ChannelVocoder7Kernel";
-		Configuration cfg = readConfiguration(String.format("%d%s", 1, appName));
+		Configuration cfg = ConfigurationUtils.readConfiguration(String.format(
+				"%d%s.cfg", 1, appName));
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(
 					String.format("%d%s.cfg", 0, appName)));
@@ -387,21 +391,6 @@ private static void printCfgValues(String fileName) {
 
 	}
 
-	private static Configuration readConfiguration(String simpeName) {
-		// String name = String.format("%s.cfg", simpeName);
-		String name = simpeName;
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			reader.close();
-			return Configuration.fromJson(json);
-		} catch (Exception ex) {
-			System.err.println(String.format(
-					"File reader error. No %s configuration file.", name));
-		}
-		return null;
-	}
-
 	private static void printAll(String folderPath) {
 		File folder = new File(folderPath);
 		File[] listOfFiles = folder.listFiles(new FilenameFilter() {

From 530b0909bd07196824045b48fe186c27fc1aab07 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 01:35:44 +0800
Subject: [PATCH 387/881] printCfgValues() uses ConfigurationUtils.

---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index ab06c6eb..7ea55481 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -378,7 +378,7 @@ private static void saveConfg(Configuration config, int round,
 	}
 
 	private static void printCfgValues(String fileName) {
-		Configuration cfg = readConfiguration(fileName);
+		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
 		File f = new File(fileName);
 		Map<String, Parameter> parameters = cfg.getParametersMap();
 		System.out.println(String.format("%s - %d", f.getName(), parameters

From d545bb43eb1ac5920b51574318c8bf5b524f6fe9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 01:42:03 +0800
Subject: [PATCH 388/881] printCfgValues() checks null condition.

readConfiguration() returns null if no file exists.
---
 .../impl/common/ConfigurationEditor.java      | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 7ea55481..c36699df 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -2,7 +2,6 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.FilenameFilter;
@@ -23,10 +22,6 @@
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
-import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
-import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
-import edu.mit.streamjit.test.apps.fmradio.FMRadio;
-import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
@@ -379,16 +374,17 @@ private static void saveConfg(Configuration config, int round,
 
 	private static void printCfgValues(String fileName) {
 		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
-		File f = new File(fileName);
-		Map<String, Parameter> parameters = cfg.getParametersMap();
-		System.out.println(String.format("%s - %d", f.getName(), parameters
-				.entrySet().size()));
-		// int i = 0;
-		//
-		// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
-		// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
-		// }
-
+		if (cfg != null) {
+			File f = new File(fileName);
+			Map<String, Parameter> parameters = cfg.getParametersMap();
+			System.out.println(String.format("%s - %d", f.getName(), parameters
+					.entrySet().size()));
+			// int i = 0;
+			//
+			// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
+			// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
+			// }
+		}
 	}
 
 	private static void printAll(String folderPath) {

From 28e98f7b49c5cac495367a7781e5c14695baf138 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 01:59:12 +0800
Subject: [PATCH 389/881] saveConfig() has been moved to ConfigurationUtils

from ConfigurationEditor.
---
 .../impl/common/ConfigurationEditor.java      | 34 ++---------------
 .../streamjit/util/ConfigurationUtils.java    | 38 +++++++++++++++++++
 2 files changed, 42 insertions(+), 30 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index c36699df..bd2ed45d 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -61,7 +61,8 @@ private static void changeMultiplierVal() {
 
 		IntParameter newMulParam = new IntParameter("multiplier", 1, 100, 100);
 		builder.addParameter(newMulParam);
-		saveConfg(builder.build(), 444, "final_NestedSplitJoinCore");
+		ConfigurationUtils.saveConfg(builder.build(), "444",
+				"final_NestedSplitJoinCore");
 
 	}
 
@@ -167,8 +168,7 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 
 		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
 
-		saveConfg(cfg, 0, name);
-
+		ConfigurationUtils.saveConfg(cfg, "0", name);
 	}
 
 	private static void edit1(String name, int maxWor)
@@ -272,7 +272,7 @@ private static void convert() {
 			String pythonDict = reader.readLine();
 
 			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
-			saveConfg(finalCfg, 0, name);
+			ConfigurationUtils.saveConfg(finalCfg, "0", name);
 		} catch (Exception ex) {
 			ex.printStackTrace();
 		}
@@ -346,32 +346,6 @@ private static String getConfigurationString(Configuration cfg) {
 		return s3;
 	}
 
-	/**
-	 * Save the configuration.
-	 */
-	private static void saveConfg(Configuration config, int round,
-			String appName) {
-		String json = config.toJson();
-		try {
-
-			File dir = new File(String.format("configurations%s%s",
-					File.separator, appName));
-			if (!dir.exists())
-				if (!dir.mkdirs()) {
-					System.err.println("Make directory failed");
-					return;
-				}
-
-			File file = new File(dir, String.format("%d%s.cfg", round, appName));
-			FileWriter writer = new FileWriter(file, false);
-			writer.write(json);
-			writer.flush();
-			writer.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-	}
-
 	private static void printCfgValues(String fileName) {
 		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
 		if (cfg != null) {
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 9970a615..8233ff31 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -1,7 +1,9 @@
 package edu.mit.streamjit.util;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 
 import edu.mit.streamjit.impl.common.Configuration;
@@ -39,4 +41,40 @@ public static Configuration readConfiguration(String cfgFilePath) {
 		}
 		return null;
 	}
+
+	/**
+	 * Saves the config into configurations/appName directory. output _.cfg file
+	 * will be named as namePrefixappName.cfg.
+	 * 
+	 * @param config
+	 *            {@link Configuration} that need to be saved.
+	 * @param namePrefix
+	 *            prefix to add to the out put file name.
+	 * @param appName
+	 *            name of the streamJit app. output _.cfg file will be named as
+	 *            namePrefixappName.cfg.
+	 */
+	public static void saveConfg(Configuration config, String namePrefix,
+			String appName) {
+		String json = config.toJson();
+		try {
+
+			File dir = new File(String.format("configurations%s%s",
+					File.separator, appName));
+			if (!dir.exists())
+				if (!dir.mkdirs()) {
+					System.err.println("Make directory failed");
+					return;
+				}
+
+			File file = new File(dir, String.format("%s%s.cfg", namePrefix,
+					appName));
+			FileWriter writer = new FileWriter(file, false);
+			writer.write(json);
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
 }

From a6c9dc4126762ef1695aaca32263ad2dc1d54ce6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 02:04:20 +0800
Subject: [PATCH 390/881] an overloaded method saveCofig() added.

saveConfig(String, String, String) has been added.
---
 .../streamjit/util/ConfigurationUtils.java    | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 8233ff31..f270fdf3 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -57,6 +57,24 @@ public static Configuration readConfiguration(String cfgFilePath) {
 	public static void saveConfg(Configuration config, String namePrefix,
 			String appName) {
 		String json = config.toJson();
+		saveConfg(json, namePrefix, appName);
+	}
+
+	/**
+	 * Saves the configJson into configurations/appName directory. output _.cfg
+	 * file will be named as namePrefixappName.cfg.
+	 * 
+	 * @param configJson
+	 *            Json representation of the {@link Configuration} that need to
+	 *            be saved.
+	 * @param namePrefix
+	 *            prefix to add to the out put file name.
+	 * @param appName
+	 *            name of the streamJit app. output _.cfg file will be named as
+	 *            namePrefixappName.cfg.
+	 */
+	public static void saveConfg(String configJson, String namePrefix,
+			String appName) {
 		try {
 
 			File dir = new File(String.format("configurations%s%s",
@@ -70,7 +88,7 @@ public static void saveConfg(Configuration config, String namePrefix,
 			File file = new File(dir, String.format("%s%s.cfg", namePrefix,
 					appName));
 			FileWriter writer = new FileWriter(file, false);
-			writer.write(json);
+			writer.write(configJson);
 			writer.flush();
 			writer.close();
 		} catch (IOException e) {

From 2653021c050ff4351e693fb5ac544d7be24278e7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 02:13:56 +0800
Subject: [PATCH 391/881] OnlineTuner uses ConfigurationUtils.saveConfig()

---
 .../distributed/runtimer/OnlineTuner.java     | 31 +++----------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8170656f..19dafc5f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
+import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
@@ -92,7 +93,8 @@ private void tune() {
 				Configuration config = Configuration.fromJson(cfgJson);
 
 				if (GlobalConstants.saveAllConfigurations)
-					saveConfg(cfgJson, round);
+					ConfigurationUtils.saveConfg(cfgJson,
+							new Integer(round).toString(), app.name);
 
 				ret = reconfigure(config);
 				if (ret.first) {
@@ -207,7 +209,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		saveConfg(finalConfg, 0);
+		ConfigurationUtils.saveConfg(finalConfg, "final_", app.name);
 
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		evaluateConfig(finalcfg, "Final configuration");
@@ -289,29 +291,4 @@ private Configuration readConfiguration(String name) {
 		}
 		return null;
 	}
-
-	/**
-	 * Save the configuration.
-	 */
-	private void saveConfg(String json, int round) {
-		try {
-
-			File dir = new File(String.format("configurations%s%s",
-					File.separator, app.name));
-			if (!dir.exists())
-				if (!dir.mkdirs()) {
-					System.err.println("Make directory failed");
-					return;
-				}
-
-			File file = new File(dir,
-					String.format("%d%s.cfg", round, app.name));
-			FileWriter writer = new FileWriter(file, false);
-			writer.write(json);
-			writer.flush();
-			writer.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-	}
 }
\ No newline at end of file

From 8fe1128ac0efc6ccd129ac7688f59da907015073 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 04:04:02 +0800
Subject: [PATCH 392/881] OnlineTuner.java uses CfgUtils to read cfgs.

ConfigurationUtils.readConfiguration()
---
 .../distributed/runtimer/OnlineTuner.java     | 25 +++++--------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 19dafc5f..12548b93 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -121,12 +121,12 @@ private void tune() {
 	}
 
 	private void evaluate() {
-		Configuration finalCfg = readConfiguration(String.format(
-				"final_%s.cfg", app.name));
+		Configuration finalCfg = ConfigurationUtils.readConfiguration(String
+				.format("final_%s.cfg", app.name));
 		evaluateConfig(finalCfg, "Final configuration");
 
-		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
-				app.name));
+		Configuration handCfg = ConfigurationUtils.readConfiguration(String
+				.format("hand_%s.cfg", app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		try {
@@ -214,8 +214,8 @@ private void handleTermination() throws IOException {
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		evaluateConfig(finalcfg, "Final configuration");
 
-		Configuration handCfg = readConfiguration(String.format("hand_%s.cfg",
-				app.name));
+		Configuration handCfg = ConfigurationUtils.readConfiguration(String
+				.format("hand_%s.cfg", app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {
@@ -278,17 +278,4 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			e1.printStackTrace();
 		}
 	}
-
-	private Configuration readConfiguration(String name) {
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			reader.close();
-			return Configuration.fromJson(json);
-		} catch (Exception ex) {
-			System.err.println(String.format(
-					"File reader error. No %s configuration file.", name));
-		}
-		return null;
-	}
 }
\ No newline at end of file

From 05db9187da3d2ea748861c4cd6a4a41aec09b35e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 25 Oct 2014 04:10:05 +0800
Subject: [PATCH 393/881] DistStrmCompiler uses cfgUtils.readConfiguration()

---
 .../DistributedStreamCompiler.java            | 21 +++----------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 1789fc5f..603083ea 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -1,7 +1,5 @@
 package edu.mit.streamjit.impl.distributed;
 
-import java.io.BufferedReader;
-import java.io.FileReader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -47,6 +45,7 @@
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
+import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
 
 /**
@@ -154,8 +153,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 	private <I, O> Configuration cfgFromFile(OneToOneElement<I, O> stream,
 			Controller controller, Configuration defaultCfg) {
-		Configuration cfg1 = readConfiguration(stream.getClass()
-				.getSimpleName());
+		Configuration cfg1 = ConfigurationUtils.readConfiguration(String
+				.format("%s.cfg", stream.getClass().getSimpleName()));
 		if (cfg1 == null) {
 			controller.closeAll();
 			throw new IllegalConfigurationException();
@@ -243,20 +242,6 @@ private <I, O> void manualPartition(
 		app.newPartitionMap(partitionsMachineMap);
 	}
 
-	private Configuration readConfiguration(String simpeName) {
-		String name = String.format("%s.cfg", simpeName);
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			reader.close();
-			return Configuration.fromJson(json);
-		} catch (Exception ex) {
-			System.err.println(String.format(
-					"File reader error. No %s configuration file.", name));
-		}
-		return null;
-	}
-
 	/**
 	 * Sets head and tail buffers.
 	 */

From c03100ab8efe4cae4f76be76a8e28bed1c4cdb45 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 26 Oct 2014 04:15:00 +0800
Subject: [PATCH 394/881] Uses ConfigUtils.readCfg() method.

---
 src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index 73f2d2a9..036f4475 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -14,6 +14,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.tuner.SqliteAdapter;
+import edu.mit.streamjit.util.ConfigurationUtils;
 
 public class ConfigurationAnalyzer {
 
@@ -157,7 +158,7 @@ private Configuration readcoConfiguration(String appDir, String appName,
 			Integer cfgNo) {
 		String cfg = String.format("%s%s%d%s.cfg", appDir, File.separator,
 				cfgNo, appName);
-		return readConfiguration(cfg);
+		return ConfigurationUtils.readConfiguration(cfg);
 	}
 
 	private Configuration readConfiguration(String name) {

From 22d8409623208a248a6730a8c84c2f578745ba2c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 26 Oct 2014 04:16:23 +0800
Subject: [PATCH 395/881] readConfiguration() removed

local method readConfiguration() has been removed as ConfigurationUtils offers
same method.
---
 .../impl/common/ConfigurationAnalyzer.java        | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index 036f4475..8e695e3c 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -1,8 +1,6 @@
 package edu.mit.streamjit.impl.common;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.LinkedList;
@@ -161,19 +159,6 @@ private Configuration readcoConfiguration(String appDir, String appName,
 		return ConfigurationUtils.readConfiguration(cfg);
 	}
 
-	private Configuration readConfiguration(String name) {
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			reader.close();
-			return Configuration.fromJson(json);
-		} catch (Exception ex) {
-			System.err.println(String.format(
-					"File reader error. No %s configuration file.", name));
-		}
-		return null;
-	}
-
 	private boolean verifyPath(String cfgDir, String appName) {
 		String dbPath = appName;
 		File db = new File(dbPath);

From 540e082603bb41ea000e04fea90e420ebeea2135 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 27 Oct 2014 00:22:24 +0800
Subject: [PATCH 396/881] BugFix: sets lastCount=0 when reset.

---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 4a7198db..74b9b59b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -69,6 +69,7 @@ public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 		this.skipCount = skipCount;
 		this.totalCount = steadyCount + skipCount;
 		count = 0;
+		lastCount = 0;
 		steadyLatch = new CountDownLatch(1);
 		skipLatch = new CountDownLatch(1);
 		this.skipLatchUp = true;
@@ -155,6 +156,7 @@ public void reset() {
 		skipLatch.countDown();
 		skipLatch = new CountDownLatch(1);
 		count = 0;
+		lastCount = 0;
 		skipLatchUp = true;
 	}
 
@@ -169,6 +171,7 @@ public void releaseAll() {
 		steadyLatch.countDown();
 		skipLatch.countDown();
 		count = 0;
+		lastCount = 0;
 	}
 
 	private class PerformanceLogger extends Thread {

From 523dae94b87c09de0ea67b29365191f91f937e18 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 14:32:29 +0800
Subject: [PATCH 397/881] Refactored : intermediateDraining() added.

---
 .../distributed/runtimer/OnlineTuner.java     | 45 ++++++++++++-------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 12548b93..6940794b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -155,21 +155,9 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				return new Pair<Boolean, Long>(true, -1l);
 			}
 
-			if (manager.isRunning()) {
-				boolean state = drainer.startDraining(0);
-				if (!state) {
-					System.err
-							.println("Final drain has already been called. no more tuning.");
-					return new Pair<Boolean, Long>(false, -1l);
-				}
-
-				System.err.println("awaitDrainedIntrmdiate");
-				drainer.awaitDrainedIntrmdiate();
-
-				drainer.awaitDrainData();
-				DrainData drainData = drainer.getDrainData();
-				app.drainData = drainData;
-			}
+			boolean isDrained = intermediateDraining();
+			if (!isDrained)
+				return new Pair<Boolean, Long>(false, -1l);
 
 			int multiplier = 1000;
 			IntParameter mulParam = config.getParameter("multiplier",
@@ -201,6 +189,33 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 		return new Pair<Boolean, Long>(true, time);
 	}
 
+	/**
+	 * Performs intermediate draining.
+	 * 
+	 * @return <code>true</code> iff the draining is success or the application
+	 *         is not running currently.
+	 * @throws InterruptedException
+	 */
+	private boolean intermediateDraining() throws InterruptedException {
+		if (manager.isRunning()) {
+			boolean state = drainer.startDraining(0);
+			if (!state) {
+				System.err
+						.println("Final drain has already been called. no more tuning.");
+				return false;
+			}
+
+			System.err.println("awaitDrainedIntrmdiate");
+			drainer.awaitDrainedIntrmdiate();
+
+			drainer.awaitDrainData();
+			DrainData drainData = drainer.getDrainData();
+			app.drainData = drainData;
+			return true;
+		} else
+			return true;
+	}
+
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 

From 0cf2d9f5c44fba680d9087faa79a9c6246c2958a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 14:37:13 +0800
Subject: [PATCH 398/881] Refactored : getMultiplier() added.

---
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 6940794b..d54b8355 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -159,14 +159,10 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			if (!isDrained)
 				return new Pair<Boolean, Long>(false, -1l);
 
-			int multiplier = 1000;
-			IntParameter mulParam = config.getParameter("multiplier",
-					IntParameter.class);
-			if (mulParam != null)
-				multiplier = mulParam.getValue();
-
 			drainer.setBlobGraph(app.blobGraph);
-			System.err.println("Reconfiguring...multiplier = " + multiplier);
+
+			int multiplier = getMultiplier(config);
+
 			if (manager.reconfigure(multiplier)) {
 				// TODO: need to check the manager's status before
 				// passing
@@ -216,6 +212,16 @@ private boolean intermediateDraining() throws InterruptedException {
 			return true;
 	}
 
+	private int getMultiplier(Configuration config) {
+		int multiplier = 1000;
+		IntParameter mulParam = config.getParameter("multiplier",
+				IntParameter.class);
+		if (mulParam != null)
+			multiplier = mulParam.getValue();
+		System.err.println("Reconfiguring...multiplier = " + multiplier);
+		return multiplier;
+	}
+
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 

From a4f03eb1049513a23c6ddc2a9ef03b68fb5a004a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 14:42:50 +0800
Subject: [PATCH 399/881] Comment formatted.

---
 .../impl/distributed/runtimer/OnlineTuner.java   | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d54b8355..ed5c73d0 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -151,24 +151,18 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			return new Pair<Boolean, Long>(false, 0l);
 
 		try {
-			if (!cfgManager.newConfiguration(config)) {
+			if (!cfgManager.newConfiguration(config))
 				return new Pair<Boolean, Long>(true, -1l);
-			}
 
-			boolean isDrained = intermediateDraining();
-			if (!isDrained)
+			if (!intermediateDraining())
 				return new Pair<Boolean, Long>(false, -1l);
 
 			drainer.setBlobGraph(app.blobGraph);
-
 			int multiplier = getMultiplier(config);
-
 			if (manager.reconfigure(multiplier)) {
-				// TODO: need to check the manager's status before
-				// passing
-				// the time. Exceptions, final drain, etc may causes app
-				// to
-				// stop executing.
+				// TODO: need to check the manager's status before passing the
+				// time. Exceptions, final drain, etc may causes app to stop
+				// executing.
 				time = manager.getFixedOutputTime();
 
 				System.out.println("Execution time is " + time

From b67303f86930ccf1b8dbaa677719327c442174f6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 15:52:48 +0800
Subject: [PATCH 400/881] Refactored : processStaticCfg() added.

---
 .../node/ConfigurationProcessorImpl.java      | 31 ++++++++++---------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 95728734..26ee7970 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -58,20 +58,7 @@ public ConfigurationProcessorImpl(StreamNode streamNode) {
 	@Override
 	public void process(String json, ConfigType type, DrainData drainData) {
 		if (type == ConfigType.STATIC) {
-			if (this.staticConfig == null) {
-				this.staticConfig = Jsonifiers.fromJson(json,
-						Configuration.class);
-
-				Map<Integer, InetAddress> iNetAddressMap = (Map<Integer, InetAddress>) staticConfig
-						.getExtraData(GlobalConstants.INETADDRESS_MAP);
-
-				NetworkInfo networkInfo = new NetworkInfo(iNetAddressMap);
-
-				this.conProvider = new ConnectionProvider(
-						streamNode.getNodeID(), networkInfo);
-			} else
-				System.err
-						.println("New static configuration received...But Ignored...");
+			processStaticCfg(json);
 		} else {
 			System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
 			System.out.println("New Configuration.....");
@@ -111,6 +98,22 @@ public void process(String json, ConfigType type, DrainData drainData) {
 		}
 	}
 
+	private void processStaticCfg(String json) {
+		if (this.staticConfig == null) {
+			this.staticConfig = Jsonifiers.fromJson(json, Configuration.class);
+
+			Map<Integer, InetAddress> iNetAddressMap = (Map<Integer, InetAddress>) staticConfig
+					.getExtraData(GlobalConstants.INETADDRESS_MAP);
+
+			NetworkInfo networkInfo = new NetworkInfo(iNetAddressMap);
+
+			this.conProvider = new ConnectionProvider(streamNode.getNodeID(),
+					networkInfo);
+		} else
+			System.err
+					.println("New static configuration received...But Ignored...");
+	}
+
 	private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			Configuration stccfg, DrainData drainData) {
 

From bafcb40277a3c306c77a7364a848824566df6162 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 15:55:51 +0800
Subject: [PATCH 401/881] Refactored : processDynamicCfg() added.

---
 .../node/ConfigurationProcessorImpl.java      | 73 ++++++++++---------
 1 file changed, 38 insertions(+), 35 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 26ee7970..534a82f9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -60,41 +60,7 @@ public void process(String json, ConfigType type, DrainData drainData) {
 		if (type == ConfigType.STATIC) {
 			processStaticCfg(json);
 		} else {
-			System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
-			System.out.println("New Configuration.....");
-			// [2014-3-20] We need to release blobsmanager to release the
-			// memory. Otherwise, Blobthread2.corecode will hold the memory.
-			BlobsManager bm = streamNode.getBlobsManager();
-			if (bm != null) {
-				bm.stop();
-				streamNode.setBlobsManager(null);
-				bm = null;
-			}
-			Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
-			ImmutableSet<Blob> blobSet = getBlobs(cfg, staticConfig, drainData);
-			if (blobSet != null) {
-				try {
-					streamNode.controllerConnection
-							.writeObject(AppStatus.COMPILED);
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-
-				Map<Token, ConnectionInfo> conInfoMap = (Map<Token, ConnectionInfo>) cfg
-						.getExtraData(GlobalConstants.CONINFOMAP);
-
-				streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
-						conInfoMap, streamNode, conProvider));
-			} else {
-				try {
-					streamNode.controllerConnection
-							.writeObject(AppStatus.COMPILATION_ERROR);
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-
-				System.out.println("Couldn't get the blobset....");
-			}
+			processDynamicCfg(json, drainData);
 		}
 	}
 
@@ -114,6 +80,43 @@ private void processStaticCfg(String json) {
 					.println("New static configuration received...But Ignored...");
 	}
 
+	private void processDynamicCfg(String json, DrainData drainData) {
+		System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
+		System.out.println("New Configuration.....");
+		// [2014-3-20] We need to release blobsmanager to release the
+		// memory. Otherwise, Blobthread2.corecode will hold the memory.
+		BlobsManager bm = streamNode.getBlobsManager();
+		if (bm != null) {
+			bm.stop();
+			streamNode.setBlobsManager(null);
+			bm = null;
+		}
+		Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
+		ImmutableSet<Blob> blobSet = getBlobs(cfg, staticConfig, drainData);
+		if (blobSet != null) {
+			try {
+				streamNode.controllerConnection.writeObject(AppStatus.COMPILED);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+
+			Map<Token, ConnectionInfo> conInfoMap = (Map<Token, ConnectionInfo>) cfg
+					.getExtraData(GlobalConstants.CONINFOMAP);
+
+			streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
+					conInfoMap, streamNode, conProvider));
+		} else {
+			try {
+				streamNode.controllerConnection
+						.writeObject(AppStatus.COMPILATION_ERROR);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+
+			System.out.println("Couldn't get the blobset....");
+		}
+	}
+
 	private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			Configuration stccfg, DrainData drainData) {
 

From df78e83c5f19693a3b71cecf317774241c2b9020 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 7 Nov 2014 16:06:45 +0800
Subject: [PATCH 402/881] Refactored : releaseOldBM() added.

---
 .../node/ConfigurationProcessorImpl.java      | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 534a82f9..0ccc025c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -83,14 +83,7 @@ private void processStaticCfg(String json) {
 	private void processDynamicCfg(String json, DrainData drainData) {
 		System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
 		System.out.println("New Configuration.....");
-		// [2014-3-20] We need to release blobsmanager to release the
-		// memory. Otherwise, Blobthread2.corecode will hold the memory.
-		BlobsManager bm = streamNode.getBlobsManager();
-		if (bm != null) {
-			bm.stop();
-			streamNode.setBlobsManager(null);
-			bm = null;
-		}
+		releaseOldBM();
 		Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
 		ImmutableSet<Blob> blobSet = getBlobs(cfg, staticConfig, drainData);
 		if (blobSet != null) {
@@ -117,6 +110,20 @@ private void processDynamicCfg(String json, DrainData drainData) {
 		}
 	}
 
+	/**
+	 * Un-references old BlobManager object before creating new one.
+	 */
+	private void releaseOldBM() {
+		// [2014-3-20] We need to release blobsmanager to release the
+		// memory. Otherwise, Blobthread2.corecode will occupy the memory.
+		BlobsManager bm = streamNode.getBlobsManager();
+		if (bm != null) {
+			bm.stop();
+			streamNode.setBlobsManager(null);
+			bm = null;
+		}
+	}
+
 	private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			Configuration stccfg, DrainData drainData) {
 

From fa698854a6ee54456e15ff17ab6d5e38c5e632fd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Nov 2014 10:57:13 +0800
Subject: [PATCH 403/881] Refactored : printMemoryStatus() added.

---
 .../impl/distributed/StreamJitAppManager.java      | 13 ++-----------
 .../streamjit/impl/distributed/common/Utils.java   | 14 ++++++++++++++
 .../impl/distributed/node/BlobsManagerImpl.java    | 14 +-------------
 3 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 6a8b3818..7b45a58e 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -39,6 +39,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 public class StreamJitAppManager {
@@ -243,18 +244,8 @@ public boolean reconfigure(int multiplier) {
 			isRunning = false;
 		}
 
-		long heapMaxSize = Runtime.getRuntime().maxMemory();
-		long heapSize = Runtime.getRuntime().totalMemory();
-		long heapFreeSize = Runtime.getRuntime().freeMemory();
-
-		System.out.println("##############Controller######################");
-
-		System.out.println("heapMaxSize = " + heapMaxSize / 1e6);
-		System.out.println("heapSize = " + heapSize / 1e6);
-		System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
 		System.out.println("StraemJit app is running...");
-		System.out.println("##############################################");
-
+		Utils.printMemoryStatus();
 		return isRunning;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 35eae525..a15d0e05 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -29,4 +29,18 @@ public static Token getblobID(Set<Worker<?, ?>> workers) {
 
 		return Collections.min(inputBuilder.build());
 	}
+
+	/**
+	 * Prints heapMaxSize, current heapSize and heapFreeSize.
+	 */
+	public static void printMemoryStatus() {
+		long heapMaxSize = Runtime.getRuntime().maxMemory();
+		long heapSize = Runtime.getRuntime().totalMemory();
+		long heapFreeSize = Runtime.getRuntime().freeMemory();
+		System.out.println("##############################################");
+		System.out.println("heapMaxSize = " + heapMaxSize / 1e6);
+		System.out.println("heapSize = " + heapSize / 1e6);
+		System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
+		System.out.println("##############################################");
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f93bd969..4aeb055d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -678,20 +678,8 @@ private class CommandProcessorImpl implements CommandProcessor {
 		@Override
 		public void processSTART() {
 			start();
-			long heapMaxSize = Runtime.getRuntime().maxMemory();
-			long heapSize = Runtime.getRuntime().totalMemory();
-			long heapFreeSize = Runtime.getRuntime().freeMemory();
-
-			System.out
-					.println("##############################################");
-
-			System.out.println("heapMaxSize = " + heapMaxSize / 1e6);
-			System.out.println("heapSize = " + heapSize / 1e6);
-			System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
 			System.out.println("StraemJit app is running...");
-			System.out
-					.println("##############################################");
-
+			Utils.printMemoryStatus();
 		}
 
 		@Override

From 41e899cfe0df578d8bbfe548ae9453118693aad0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Nov 2014 11:03:43 +0800
Subject: [PATCH 404/881] Refactored : printOutOfMemory() added.

---
 .../streamjit/impl/distributed/common/Utils.java   | 14 ++++++++++++++
 .../node/ConfigurationProcessorImpl.java           | 14 ++------------
 src/edu/mit/streamjit/tuner/RunApp.java            | 13 ++-----------
 3 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index a15d0e05..69238644 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -1,5 +1,8 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
 import java.util.Collections;
 import java.util.Set;
 
@@ -43,4 +46,15 @@ public static void printMemoryStatus() {
 		System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
 		System.out.println("##############################################");
 	}
+
+	public static void printOutOfMemory() {
+		MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
+		System.out.println("******OutOfMemoryError******");
+		MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
+		int MEGABYTE = 1024 * 1024;
+		long maxMemory = heapUsage.getMax() / MEGABYTE;
+		long usedMemory = heapUsage.getUsed() / MEGABYTE;
+		System.out
+				.println("Memory Use :" + usedMemory + "M/" + maxMemory + "M");
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 0ccc025c..7220a9a7 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -4,9 +4,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.MemoryUsage;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -35,6 +32,7 @@
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -168,15 +166,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					ex.printStackTrace();
 					return null;
 				} catch (OutOfMemoryError er) {
-					MemoryMXBean memoryBean = ManagementFactory
-							.getMemoryMXBean();
-					System.out.println("******OutOfMemoryError******");
-					MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
-					int MEGABYTE = 1024 * 1024;
-					long maxMemory = heapUsage.getMax() / MEGABYTE;
-					long usedMemory = heapUsage.getUsed() / MEGABYTE;
-					System.out.println("Memory Use :" + usedMemory + "M/"
-							+ maxMemory + "M");
+					Utils.printOutOfMemory();
 					return null;
 				}
 				// DEBUG MSG
diff --git a/src/edu/mit/streamjit/tuner/RunApp.java b/src/edu/mit/streamjit/tuner/RunApp.java
index f0a0829b..3bf541b0 100644
--- a/src/edu/mit/streamjit/tuner/RunApp.java
+++ b/src/edu/mit/streamjit/tuner/RunApp.java
@@ -1,8 +1,5 @@
 package edu.mit.streamjit.tuner;
 
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.MemoryUsage;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
@@ -11,6 +8,7 @@
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
 
@@ -83,14 +81,7 @@ else if (benchmarkResult.kind() == Benchmarker.Result.Kind.EXCEPTION) {
 			e.printStackTrace();
 			time = -2;
 		} catch (OutOfMemoryError er) {
-			MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
-			System.out.println("******OutOfMemoryError******");
-			MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
-			int MEGABYTE = 1024 * 1024;
-			long maxMemory = heapUsage.getMax() / MEGABYTE;
-			long usedMemory = heapUsage.getUsed() / MEGABYTE;
-			System.out.println("Memory Use :" + usedMemory + "M/" + maxMemory
-					+ "M");
+			Utils.printOutOfMemory();
 			time = -3;
 		}
 

From f97bc024035eac3a377187bcf3ba8df0cc1657bc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Nov 2014 14:50:06 +0800
Subject: [PATCH 405/881] TuningStatistics added

This new independent class goes through tuning results and prints statistics.
---
 .../impl/common/ConfigurationEditor.java      | 32 -------
 .../impl/common/TuningStatistics.java         | 92 +++++++++++++++++++
 2 files changed, 92 insertions(+), 32 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/common/TuningStatistics.java

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index bd2ed45d..e196e148 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -40,8 +40,6 @@ public static void main(String[] args) throws IOException {
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
 		// changeMultiplierVal();
-		// printCfgValues("1NestedSplitJoinCore.cfg");
-		printAll("../Tuner layer/tuning-oopsla2014");
 	}
 
 	/**
@@ -345,34 +343,4 @@ private static String getConfigurationString(Configuration cfg) {
 		String s3 = s2.replaceAll("ttttt", "__class__");
 		return s3;
 	}
-
-	private static void printCfgValues(String fileName) {
-		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
-		if (cfg != null) {
-			File f = new File(fileName);
-			Map<String, Parameter> parameters = cfg.getParametersMap();
-			System.out.println(String.format("%s - %d", f.getName(), parameters
-					.entrySet().size()));
-			// int i = 0;
-			//
-			// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
-			// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
-			// }
-		}
-	}
-
-	private static void printAll(String folderPath) {
-		File folder = new File(folderPath);
-		File[] listOfFiles = folder.listFiles(new FilenameFilter() {
-
-			@Override
-			public boolean accept(File dir, String name) {
-				return name.toLowerCase().endsWith(".cfg");
-			}
-		});
-		System.out.println(String.format("Parameters in configuration..."));
-		for (int i = 0; i < listOfFiles.length; i++) {
-			printCfgValues(listOfFiles[i].getAbsolutePath());
-		}
-	}
 }
diff --git a/src/edu/mit/streamjit/impl/common/TuningStatistics.java b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
new file mode 100644
index 00000000..8003c4ba
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
@@ -0,0 +1,92 @@
+package edu.mit.streamjit.impl.common;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Map;
+
+import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
+import edu.mit.streamjit.util.ConfigurationUtils;
+
+public class TuningStatistics {
+
+	private static String[] params = { "remove", "fuse", "unboxStorage",
+			"unboxInput", "unboxOutput" };
+
+	/**
+	 * @param args
+	 * @throws IOException
+	 */
+	public static void main(String[] args) throws IOException {
+		// printCfgValues("1NestedSplitJoinCore.cfg");
+		printAll("../Tuner layer/tuning-oopsla2014");
+
+	}
+
+	private static double getBoolParamStat(Map<String, Parameter> parameters,
+			String prefix) {
+		int totalParams = 0;
+		int noOfTRUE = 0;
+
+		for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
+			if (en.getKey().startsWith(prefix)) {
+				totalParams++;
+				SwitchParameter<Boolean> p = (SwitchParameter<Boolean>) en
+						.getValue();
+				if (p.getValue()) {
+					noOfTRUE++;
+				}
+			}
+		}
+		double per = 100 * (double) noOfTRUE / totalParams;
+		System.out.println("totalParams - " + totalParams);
+		System.out.println("noOfTRUE - " + noOfTRUE);
+		System.out.println("Percentage - " + per);
+		return per;
+	}
+
+	private static void printAll(String folderPath) throws IOException {
+		File folder = new File(folderPath);
+		File[] listOfFiles = folder.listFiles(new FilenameFilter() {
+
+			@Override
+			public boolean accept(File dir, String name) {
+				return name.toLowerCase().endsWith(".cfg");
+			}
+		});
+		System.out.println(String.format("Parameters in configuration..."));
+
+		FileWriter writer = new FileWriter("paramStat.dat", false);
+		writeHeader(writer);
+		for (int i = 0; i < listOfFiles.length; i++) {
+			printCfgValues(listOfFiles[i].getAbsolutePath());
+		}
+	}
+
+	private static void printCfgValues(String fileName) {
+		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
+		if (cfg != null) {
+			File f = new File(fileName);
+			Map<String, Parameter> parameters = cfg.getParametersMap();
+			System.out.println(String.format("%s - %d", f.getName(), parameters
+					.entrySet().size()));
+			// int i = 0;
+			//
+			// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
+			// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
+			// }
+			getBoolParamStat(parameters, "unboxOutput");
+		}
+	}
+
+	private static void writeHeader(FileWriter writer) throws IOException {
+		writer.write("\t\t");
+		for (int i = 0; i < params.length; i++) {
+			writer.write("\t");
+			writer.write(params[i]);
+		}
+		writer.flush();
+	}
+}

From 32678a3103bd756b7630b92ebfb57f4836d7b567 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 11 Nov 2014 15:02:12 +0800
Subject: [PATCH 406/881] Writes statistics into a file.

Finds the statistics of boolean switch parameters and writes into a file.
---
 .../impl/common/TuningStatistics.java         | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TuningStatistics.java b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
index 8003c4ba..1545b0ad 100644
--- a/src/edu/mit/streamjit/impl/common/TuningStatistics.java
+++ b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
@@ -12,7 +12,7 @@
 
 public class TuningStatistics {
 
-	private static String[] params = { "remove", "fuse", "unboxStorage",
+	private static String[] boolParams = { "remove", "fuse", "unboxStorage",
 			"unboxInput", "unboxOutput" };
 
 	/**
@@ -25,7 +25,7 @@ public static void main(String[] args) throws IOException {
 
 	}
 
-	private static double getBoolParamStat(Map<String, Parameter> parameters,
+	private static Double getBoolParamStat(Map<String, Parameter> parameters,
 			String prefix) {
 		int totalParams = 0;
 		int noOfTRUE = 0;
@@ -61,31 +61,35 @@ public boolean accept(File dir, String name) {
 		FileWriter writer = new FileWriter("paramStat.dat", false);
 		writeHeader(writer);
 		for (int i = 0; i < listOfFiles.length; i++) {
-			printCfgValues(listOfFiles[i].getAbsolutePath());
+			printCfgValues(listOfFiles[i].getAbsolutePath(), writer);
 		}
 	}
 
-	private static void printCfgValues(String fileName) {
+	private static void printCfgValues(String fileName, FileWriter writer)
+			throws IOException {
 		Configuration cfg = ConfigurationUtils.readConfiguration(fileName);
 		if (cfg != null) {
 			File f = new File(fileName);
 			Map<String, Parameter> parameters = cfg.getParametersMap();
+			writer.write("\n");
+			writer.write(f.getName());
 			System.out.println(String.format("%s - %d", f.getName(), parameters
 					.entrySet().size()));
-			// int i = 0;
-			//
-			// for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
-			// System.out.println(String.format("\t %d.%s", i++, en.getKey()));
-			// }
-			getBoolParamStat(parameters, "unboxOutput");
+
+			for (int i = 0; i < boolParams.length; i++) {
+				writer.write("\t");
+				writer.write(getBoolParamStat(parameters, boolParams[i])
+						.toString());
+			}
+			writer.flush();
 		}
 	}
 
 	private static void writeHeader(FileWriter writer) throws IOException {
 		writer.write("\t\t");
-		for (int i = 0; i < params.length; i++) {
+		for (int i = 0; i < boolParams.length; i++) {
 			writer.write("\t");
-			writer.write(params[i]);
+			writer.write(boolParams[i]);
 		}
 		writer.flush();
 	}

From 076e7c7c3e9201438d247cee5f96454c1cfa7548 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Nov 2014 11:55:55 +0800
Subject: [PATCH 407/881] getIntParamStat() added.

Calculates statistics of IntParameters.
---
 .../impl/common/TuningStatistics.java         | 49 ++++++++++++++-----
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TuningStatistics.java b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
index 1545b0ad..6c1474d3 100644
--- a/src/edu/mit/streamjit/impl/common/TuningStatistics.java
+++ b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
@@ -6,15 +6,18 @@
 import java.io.IOException;
 import java.util.Map;
 
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
 public class TuningStatistics {
 
-	private static String[] boolParams = { "remove", "fuse", "unboxStorage",
+	private static String[] boolParams = { "remove", "fuse", "unboxStor",
 			"unboxInput", "unboxOutput" };
 
+	private static String[] intParams = { "InitBuffer", "multipl", "UnrollCo" };
+
 	/**
 	 * @param args
 	 * @throws IOException
@@ -41,12 +44,27 @@ private static Double getBoolParamStat(Map<String, Parameter> parameters,
 			}
 		}
 		double per = 100 * (double) noOfTRUE / totalParams;
-		System.out.println("totalParams - " + totalParams);
-		System.out.println("noOfTRUE - " + noOfTRUE);
-		System.out.println("Percentage - " + per);
+		// System.out.println("totalParams - " + totalParams);
+		// System.out.println("noOfTRUE - " + noOfTRUE);
+		// System.out.println("Percentage - " + per);
 		return per;
 	}
 
+	private static Integer getIntParamStat(Map<String, Parameter> parameters,
+			String prefix) {
+		int totalParams = 0;
+		int totalVal = 0;
+
+		for (Map.Entry<String, Parameter> en : parameters.entrySet()) {
+			if (en.getKey().startsWith(prefix)) {
+				totalParams++;
+				IntParameter p = (IntParameter) en.getValue();
+				totalVal += p.getValue();
+			}
+		}
+		return totalVal / totalParams;
+	}
+
 	private static void printAll(String folderPath) throws IOException {
 		File folder = new File(folderPath);
 		File[] listOfFiles = folder.listFiles(new FilenameFilter() {
@@ -72,24 +90,33 @@ private static void printCfgValues(String fileName, FileWriter writer)
 			File f = new File(fileName);
 			Map<String, Parameter> parameters = cfg.getParametersMap();
 			writer.write("\n");
-			writer.write(f.getName());
+			// writer.write(String.format("\n%.20s", f.getName()));
 			System.out.println(String.format("%s - %d", f.getName(), parameters
 					.entrySet().size()));
 
 			for (int i = 0; i < boolParams.length; i++) {
-				writer.write("\t");
-				writer.write(getBoolParamStat(parameters, boolParams[i])
-						.toString());
+				writer.write(String.format("%.2f\t\t",
+						getBoolParamStat(parameters, boolParams[i])));
 			}
+
+			for (int i = 0; i < intParams.length; i++) {
+				writer.write(String.format("%d\t\t",
+						getIntParamStat(parameters, intParams[i])));
+			}
+
 			writer.flush();
 		}
 	}
 
 	private static void writeHeader(FileWriter writer) throws IOException {
-		writer.write("\t\t");
+		// writer.write("\t\t");
 		for (int i = 0; i < boolParams.length; i++) {
-			writer.write("\t");
-			writer.write(boolParams[i]);
+			writer.write(String.format("%.7s", boolParams[i]));
+			writer.write("\t\t");
+		}
+		for (int i = 0; i < intParams.length; i++) {
+			writer.write(String.format("%.7s", intParams[i]));
+			writer.write("\t\t");
 		}
 		writer.flush();
 	}

From 86daed6978b302fd8bfcd5500c19a04c9ea1bede Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Nov 2014 11:59:01 +0800
Subject: [PATCH 408/881] getIntParamStat() scales the values to 100.

PCA is sensitive to scaling. It is better to keep all variables in same unit.
Here I scale all variables to 100.
---
 src/edu/mit/streamjit/impl/common/TuningStatistics.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/TuningStatistics.java b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
index 6c1474d3..0d831738 100644
--- a/src/edu/mit/streamjit/impl/common/TuningStatistics.java
+++ b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
@@ -59,7 +59,10 @@ private static Integer getIntParamStat(Map<String, Parameter> parameters,
 			if (en.getKey().startsWith(prefix)) {
 				totalParams++;
 				IntParameter p = (IntParameter) en.getValue();
-				totalVal += p.getValue();
+				int normalized = (100 * p.getValue())
+						/ (p.getMax() - p.getMax());
+				// totalVal += p.getValue();
+				totalVal += normalized;
 			}
 		}
 		return totalVal / totalParams;

From 2f8b744158faf5463ff93debe8a86fd0eb7fd05a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 12 Nov 2014 13:38:54 +0800
Subject: [PATCH 409/881] Refactored : Arguments removed to getStreamGraph()

getStreamGraph() gets topStreamClassName and jarFilePath directly from
static configuration.
---
 .../node/ConfigurationProcessorImpl.java      | 22 ++++++++-----------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 7220a9a7..7b245dc1 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -83,7 +83,7 @@ private void processDynamicCfg(String json, DrainData drainData) {
 		System.out.println("New Configuration.....");
 		releaseOldBM();
 		Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
-		ImmutableSet<Blob> blobSet = getBlobs(cfg, staticConfig, drainData);
+		ImmutableSet<Blob> blobSet = getBlobs(cfg, drainData);
 		if (blobSet != null) {
 			try {
 				streamNode.controllerConnection.writeObject(AppStatus.COMPILED);
@@ -123,7 +123,7 @@ private void releaseOldBM() {
 	}
 
 	private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
-			Configuration stccfg, DrainData drainData) {
+			DrainData drainData) {
 
 		PartitionParameter partParam = dyncfg.getParameter(
 				GlobalConstants.PARTITION, PartitionParameter.class);
@@ -131,13 +131,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			throw new IllegalArgumentException(
 					"Partition parameter is not available in the received configuraion");
 
-		String topLevelWorkerName = (String) stccfg
-				.getExtraData(GlobalConstants.TOPLEVEL_WORKER_NAME);
-		String jarFilePath = (String) stccfg
-				.getExtraData(GlobalConstants.JARFILE_PATH);
-
-		OneToOneElement<?, ?> streamGraph = getStreamGraph(jarFilePath,
-				topLevelWorkerName);
+		OneToOneElement<?, ?> streamGraph = getStreamGraph();
 		if (streamGraph != null) {
 			ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
 			streamGraph.visit(primitiveConnector);
@@ -184,12 +178,14 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 	/**
 	 * Gets a Stream Graph from a jar file.
 	 * 
-	 * @param jarFilePath
-	 * @param topStreamClassName
 	 * @return : StreamGraph
 	 */
-	private OneToOneElement<?, ?> getStreamGraph(String jarFilePath,
-			String topStreamClassName) {
+	private OneToOneElement<?, ?> getStreamGraph() {
+		String topStreamClassName = (String) staticConfig
+				.getExtraData(GlobalConstants.TOPLEVEL_WORKER_NAME);
+		String jarFilePath = (String) staticConfig
+				.getExtraData(GlobalConstants.JARFILE_PATH);
+
 		checkNotNull(jarFilePath);
 		checkNotNull(topStreamClassName);
 		jarFilePath = this.getClass().getProtectionDomain().getCodeSource()

From 3e38574082645d9c7f80fb307deedbfc4fe475c6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 20 Nov 2014 12:50:54 +0800
Subject: [PATCH 410/881] Comments modified/Improved.

---
 .../runtimer/BlockingCommunicationManager.java     |  2 +-
 .../impl/distributed/runtimer/StreamNodeAgent.java | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
index c9a08847..35de4f36 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/BlockingCommunicationManager.java
@@ -177,7 +177,7 @@ public InetAddress getAddress() {
 	/**
 	 * IO thread that runs a {@link StreamNodeAgent}. Since this is blocking IO
 	 * context, each {@link StreamNodeAgent} agent will be running on individual
-	 * threaed.
+	 * thread.
 	 * 
 	 */
 	private static class SNAgentRunner extends Thread {
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index 0d1e8ae5..430a263f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -48,9 +48,9 @@ public abstract class StreamNodeAgent {
 
 	private final SystemInfoProcessor sp;
 
-	// TODO: How to avoid volatile here. Because we set only once and read
-	// forever later. So if it is volatile, every read will need to access
-	// memory. Is there any way to avoid this?
+	// TODO: How to avoid the volatileness here. Because we set only once and
+	// read forever later. So if it is a volatile, every read will need to
+	// access the memory. Is there any way to avoid this?
 	// Will removing volatile modifier be OK in this context? consider
 	// using piggybacking sync or atomicreferenc with compareandset. This is
 	// actually effectively immutable/safe publication case. But how to
@@ -125,9 +125,11 @@ public NodeInfo getNodeInfo() {
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
-			// TODO: If in any chance the IO thread call this function then
-			// it will get blocked on this loop forever. Need to handle
-			// this.
+			// TODO: By any chance, if the IO thread (SNAgentRunner) calls this
+			// function then that thread will get blocked at this loop forever.
+			// Because that thread is responsible to read the nodeInfo from the
+			// StreamNode and set the nodeInfo variable of this class. Need to
+			// handle this issue.
 			while (nodeInfo == null) {
 				try {
 					Thread.sleep(10);

From 56c8b0f8a634e9188a9969b65c2a3face8d718df Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Nov 2014 11:51:22 +0800
Subject: [PATCH 411/881] Prints compilation time for each blob.

StreamNodes send compilation time information to Controller and it prints it.
---
 .../distributed/common/SNMessageVisitor.java  |  2 +
 .../impl/distributed/common/SNTimeInfo.java   | 82 +++++++++++++++++++
 .../node/ConfigurationProcessorImpl.java      |  9 ++
 .../distributed/runtimer/StreamNodeAgent.java |  9 ++
 4 files changed, 102 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java b/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
index 975669ba..c4b38957 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
@@ -13,4 +13,6 @@ public interface SNMessageVisitor {
 	void visit(SNDrainElement snDrainElement);
 
 	void visit(SNException snException);
+
+	void visit(SNTimeInfo timeInfo);
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
new file mode 100644
index 00000000..e6a220b2
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
@@ -0,0 +1,82 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
+
+/**
+ * {@link StreamNode}s shall send the timing information such as compilation
+ * time of each blob, draining time, draindata collection time, Init schedule
+ * time and etc by sending {@link SNTimeInfo}.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Nov 20, 2014
+ * 
+ */
+public abstract class SNTimeInfo implements SNMessageElement {
+
+	private static final long serialVersionUID = 1L;
+
+	public abstract void process(SNTimeInfoProcessor snTimeInfoProcessor);
+
+	@Override
+	public void accept(SNMessageVisitor visitor) {
+		visitor.visit(this);
+	}
+
+	public static final class CompilationTime extends SNTimeInfo {
+
+		private static final long serialVersionUID = 1L;
+
+		public final Token blobID;
+
+		public final double milliSec;
+
+		public CompilationTime(Token blobID, double milliSec) {
+			this.blobID = blobID;
+			this.milliSec = milliSec;
+		}
+
+		@Override
+		public void process(SNTimeInfoProcessor snTimeInfoProcessor) {
+			snTimeInfoProcessor.process(this);
+		}
+
+	}
+
+	public interface SNTimeInfoProcessor {
+		public void process(CompilationTime compilationTime);
+	}
+
+	public static class SNTimeInfoProcessorImpl implements SNTimeInfoProcessor {
+
+		private final OutputStreamWriter osw;
+
+		public SNTimeInfoProcessorImpl(OutputStream outputStream) {
+			this(new OutputStreamWriter(outputStream));
+		}
+
+		public SNTimeInfoProcessorImpl() {
+			this(System.out);
+		}
+
+		public SNTimeInfoProcessorImpl(OutputStreamWriter osw) {
+			this.osw = osw;
+		}
+
+		@Override
+		public void process(CompilationTime compilationTime) {
+			String msg = String.format("Compilation time: Blob-%s-%.0fms\n",
+					compilationTime.blobID, compilationTime.milliSec);
+			try {
+				osw.write(msg);
+				osw.flush();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 7b245dc1..c09c0cff 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -11,7 +11,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 
@@ -32,6 +34,7 @@
 import edu.mit.streamjit.impl.distributed.common.Error;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.NetworkInfo;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.CompilationTime;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
@@ -153,8 +156,14 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 				try {
 					BlobFactory bf = bs.getBlobFactory();
+					Stopwatch sw = Stopwatch.createStarted();
 					Blob b = bf.makeBlob(workerset, blobConfigs,
 							GlobalConstants.maxNumCores, drainData);
+					sw.stop();
+					CompilationTime ct = new CompilationTime(
+							Utils.getblobID(workerset),
+							sw.elapsed(TimeUnit.MILLISECONDS));
+					streamNode.controllerConnection.writeObject(ct);
 					blobSet.add(b);
 				} catch (Exception ex) {
 					ex.printStackTrace();
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index 430a263f..25549b8c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -16,7 +16,10 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessorImpl;
 import edu.mit.streamjit.impl.distributed.common.SNMessageVisitor;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo.SystemInfoProcessor;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
@@ -287,5 +290,11 @@ public void visit(SNException snException) {
 			SNExceptionProcessor snExP = manager.exceptionProcessor();
 			snException.process(snExP);
 		}
+
+		@Override
+		public void visit(SNTimeInfo timeInfo) {
+			SNTimeInfoProcessor snTIP = new SNTimeInfoProcessorImpl();
+			timeInfo.process(snTIP);
+		}
 	}
 }
\ No newline at end of file

From 1b396f59269544888d341c50d4edff04b5722ab1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 21 Nov 2014 18:35:50 +0800
Subject: [PATCH 412/881] SNTimeInfoProcessrImpl has been moved to a new file

---
 .../impl/distributed/common/SNTimeInfo.java   | 33 --------
 .../common/SNTimeInfoProcessorImpl.java       | 75 +++++++++++++++++++
 .../distributed/runtimer/StreamNodeAgent.java |  4 -
 3 files changed, 75 insertions(+), 37 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
index e6a220b2..6da090f0 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
@@ -1,9 +1,5 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 
@@ -50,33 +46,4 @@ public void process(SNTimeInfoProcessor snTimeInfoProcessor) {
 	public interface SNTimeInfoProcessor {
 		public void process(CompilationTime compilationTime);
 	}
-
-	public static class SNTimeInfoProcessorImpl implements SNTimeInfoProcessor {
-
-		private final OutputStreamWriter osw;
-
-		public SNTimeInfoProcessorImpl(OutputStream outputStream) {
-			this(new OutputStreamWriter(outputStream));
-		}
-
-		public SNTimeInfoProcessorImpl() {
-			this(System.out);
-		}
-
-		public SNTimeInfoProcessorImpl(OutputStreamWriter osw) {
-			this.osw = osw;
-		}
-
-		@Override
-		public void process(CompilationTime compilationTime) {
-			String msg = String.format("Compilation time: Blob-%s-%.0fms\n",
-					compilationTime.blobID, compilationTime.milliSec);
-			try {
-				osw.write(msg);
-				osw.flush();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
new file mode 100644
index 00000000..6b3cdc21
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
@@ -0,0 +1,75 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import java.io.FileWriter;
+import java.io.IOException;
+
+import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.CompilationTime;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
+
+public class SNTimeInfoProcessorImpl implements SNTimeInfoProcessor {
+
+	private final FileWriter compileTimeWriter;
+
+	public SNTimeInfoProcessorImpl() {
+		compileTimeWriter = getFileWriter("CompileTime.txt");
+	}
+
+	private FileWriter getFileWriter(String name) {
+		FileWriter fw = null;
+		try {
+			fw = new FileWriter(name);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return fw;
+	}
+
+	/**
+	 * {@link StreamJitAppManager} can call this at every reconfiguration to
+	 * update time info log files.
+	 * 
+	 * @param reconfigNo
+	 */
+	public void reconfigNO(int reconfigNo) {
+		if (compileTimeWriter != null) {
+			try {
+				compileTimeWriter
+						.write(String
+								.format("-------------------------%d-------------------------\n",
+										reconfigNo));
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	/**
+	 * The total compile time from {@link StreamJitAppManager} point.
+	 * 
+	 * @param time
+	 */
+	public void totalCompileTime(long time) {
+		if (compileTimeWriter != null) {
+			try {
+				compileTimeWriter.write(String.format(
+						"Total compile time %dms\n", time));
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	@Override
+	public void process(CompilationTime compilationTime) {
+		if (compilationTime != null) {
+			String msg = String.format("Blob-%s-%.0fms\n",
+					compilationTime.blobID, compilationTime.milliSec);
+			try {
+				compileTimeWriter.write(msg);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index 25549b8c..bfd0ac29 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -16,10 +16,8 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
-import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessorImpl;
 import edu.mit.streamjit.impl.distributed.common.SNMessageVisitor;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
-import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo.SystemInfoProcessor;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
@@ -293,8 +291,6 @@ public void visit(SNException snException) {
 
 		@Override
 		public void visit(SNTimeInfo timeInfo) {
-			SNTimeInfoProcessor snTIP = new SNTimeInfoProcessorImpl();
-			timeInfo.process(snTIP);
 		}
 	}
 }
\ No newline at end of file

From 2fcf5f6eae5e8e0717488efe3d926259a784ea6f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Nov 2014 17:04:01 +0800
Subject: [PATCH 413/881] TimeLogger has been added.

StreamJitAppManager, OnlineTuner, AbstractDrainer and other main classes will
use TimeLogger to log time measurements.
---
 .../mit/streamjit/impl/common/TimeLogger.java |  52 ++++++
 .../DistributedStreamCompiler.java            |   6 +-
 .../impl/distributed/StreamJitAppManager.java |   6 +-
 .../impl/distributed/TimeLoggers.java         | 152 ++++++++++++++++++
 .../distributed/runtimer/OnlineTuner.java     |  12 +-
 5 files changed, 219 insertions(+), 9 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/common/TimeLogger.java
 create mode 100644 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
new file mode 100644
index 00000000..d5276d3b
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -0,0 +1,52 @@
+package edu.mit.streamjit.impl.common;
+
+import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+
+/**
+ * Logs various time measurements for off line performance analysis. Controller
+ * node can measure the time durations for different events and use this
+ * interface to log those values.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Nov 22, 2014
+ */
+public interface TimeLogger {
+
+	/**
+	 * This method shall be called to indicate the logger that a new
+	 * configuration has been received. Appropriate caller would be
+	 * {@link OnlineTuner}.
+	 */
+	public void newReconfiguration();
+
+	/**
+	 * Log the total compilation time of a new configuration. (Controller node
+	 * point of view).
+	 * 
+	 * @param time
+	 */
+	public void logCompileTime(long time);
+
+	/**
+	 * Log the time taken to generate fixed amount of steady state outputs.
+	 * 
+	 * @param time
+	 */
+	public void logRunTime(long time);
+
+	/**
+	 * Log total draining time.
+	 * 
+	 * @param time
+	 */
+	public void logDrainTime(long time);
+
+	/**
+	 * Log total {@link DrainData} collection time.
+	 * 
+	 * @param time
+	 */
+	public void logDrainDataCollectionTime(long time);
+
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 603083ea..98c76e6a 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -33,6 +33,7 @@
 import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.OutputBufferFactory;
 import edu.mit.streamjit.impl.common.Portals;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
@@ -131,8 +132,9 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		setConfiguration(controller, srcSink, stream, app, cfgManager,
 				conManager);
 
+		TimeLogger logger = new TimeLoggers.FileTimeLogger();
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
-				cfgManager, conManager);
+				cfgManager, conManager, logger);
 		final AbstractDrainer drainer = new DistributedDrainer(manager);
 		drainer.setBlobGraph(app.blobGraph);
 
@@ -145,7 +147,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
-					cfgManager, needTermination);
+					cfgManager, logger, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
 		}
 		return cs;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 7b45a58e..8d7b8a10 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -17,6 +17,7 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPConnectionInfo;
@@ -95,12 +96,15 @@ public class StreamJitAppManager {
 	 */
 	AtomicReference<Stopwatch> stopwatchRef = new AtomicReference<>();
 
+	private final TimeLogger logger;
+
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
-			ConfigurationManager cfgManager, ConnectionManager conManager) {
+			ConfigurationManager cfgManager, ConnectionManager conManager, TimeLogger logger) {
 		this.controller = controller;
 		this.app = app;
 		this.cfgManager = cfgManager;
 		this.conManager = conManager;
+		this.logger = logger;
 		this.status = AppStatus.NOT_STARTED;
 		this.exP = new SNExceptionProcessorImpl();
 		this.ep = new ErrorProcessorImpl();
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
new file mode 100644
index 00000000..4c6a1e40
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -0,0 +1,152 @@
+package edu.mit.streamjit.impl.distributed;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import edu.mit.streamjit.impl.common.TimeLogger;
+
+/**
+ * Collection of various {@link TimeLogger} implementations.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Nov 24, 2014
+ * 
+ */
+public class TimeLoggers {
+
+	/**
+	 * Logs nothing.
+	 */
+	public static class NoTimeLogger implements TimeLogger {
+
+		@Override
+		public void newReconfiguration() {
+		}
+
+		@Override
+		public void logCompileTime(long time) {
+		}
+
+		@Override
+		public void logRunTime(long time) {
+		}
+
+		@Override
+		public void logDrainTime(long time) {
+		}
+
+		@Override
+		public void logDrainDataCollectionTime(long time) {
+		}
+
+	}
+
+	private static class TimeLoggerImpl implements TimeLogger {
+
+		private final OutputStreamWriter compileTimeWriter;
+
+		private final OutputStreamWriter runTimeWriter;
+
+		private final OutputStreamWriter drainTimeWriter;
+
+		private int reconfigNo = 0;
+
+		TimeLoggerImpl(OutputStreamWriter compileW, OutputStreamWriter runW,
+				OutputStreamWriter drainW) {
+			compileTimeWriter = compileW;
+			runTimeWriter = runW;
+			drainTimeWriter = drainW;
+		}
+
+		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
+				OutputStream drainOs) {
+			compileTimeWriter = getOSWriter(compileOS);
+			runTimeWriter = getOSWriter(runOs);
+			drainTimeWriter = getOSWriter(drainOs);
+		}
+
+		private OutputStreamWriter getOSWriter(OutputStream os) {
+			if (os == null)
+				return null;
+			return new OutputStreamWriter(os);
+		}
+
+		@Override
+		public void newReconfiguration() {
+			reconfigNo++;
+			String msg = String
+					.format("----------------------------%d----------------------------",
+							reconfigNo);
+			write(compileTimeWriter, msg);
+			write(runTimeWriter, msg);
+			write(drainTimeWriter, msg);
+		}
+
+		private void write(OutputStreamWriter osWriter, String msg) {
+			if (osWriter != null) {
+				try {
+					osWriter.write(msg);
+					osWriter.flush();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+
+		@Override
+		public void logCompileTime(long time) {
+			write(compileTimeWriter,
+					String.format("Total compile time %dms\n", time));
+		}
+
+		@Override
+		public void logRunTime(long time) {
+			write(runTimeWriter,
+					String.format("Execution time is %dms\n", time));
+		}
+
+		@Override
+		public void logDrainTime(long time) {
+			write(drainTimeWriter, String.format("Drain time is %dms\n", time));
+		}
+
+		@Override
+		public void logDrainDataCollectionTime(long time) {
+			write(drainTimeWriter,
+					String.format("Drain data collection time is %dms\n", time));
+		}
+
+	}
+
+	/**
+	 * Prints the values to the StdOut.
+	 * 
+	 */
+	public static class PrintTimeLogger extends TimeLoggerImpl {
+
+		public PrintTimeLogger() {
+			super(System.out, System.out, System.out);
+		}
+	}
+
+	public static class FileTimeLogger extends TimeLoggerImpl {
+
+		FileTimeLogger() {
+			super(getFileWriter("compileTime.txt"),
+					getFileWriter("runTime.txt"),
+					getFileWriter("drainTime.txt"));
+		}
+
+		private static FileWriter getFileWriter(String name) {
+			FileWriter fw = null;
+			try {
+				fw = new FileWriter(name);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			return fw;
+		}
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ed5c73d0..028d4e16 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,8 +1,6 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 
@@ -10,6 +8,7 @@
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
@@ -34,16 +33,18 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitApp app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
+	private final TimeLogger logger;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp app, ConfigurationManager cfgManager,
-			boolean needTermination) {
+			TimeLogger logger, boolean needTermination) {
 		this.drainer = drainer;
 		this.manager = manager;
 		this.app = app;
 		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
+		this.logger = logger;
 	}
 
 	@Override
@@ -157,6 +158,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			if (!intermediateDraining())
 				return new Pair<Boolean, Long>(false, -1l);
 
+			logger.newReconfiguration();
 			drainer.setBlobGraph(app.blobGraph);
 			int multiplier = getMultiplier(config);
 			if (manager.reconfigure(multiplier)) {
@@ -164,9 +166,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				// time. Exceptions, final drain, etc may causes app to stop
 				// executing.
 				time = manager.getFixedOutputTime();
-
-				System.out.println("Execution time is " + time
-						+ " milli seconds");
+				logger.logRunTime(time);
 			} else {
 				time = -1l;
 			}

From 28f5826c0525e8dc46a40b61a5f7c52bc08aaa6f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Nov 2014 17:28:40 +0800
Subject: [PATCH 414/881] TimeLogger: Some more methods have been added.

New methods to writes additional messages to different OutputStreamWriters.
SNTimeInfoProcessor may use this method to log additional messages those are
collected from StreamNodes.
---
 .../mit/streamjit/impl/common/TimeLogger.java | 27 +++++++++++++++++++
 .../impl/distributed/TimeLoggers.java         | 27 +++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index d5276d3b..137178ec 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -1,6 +1,7 @@
 package edu.mit.streamjit.impl.common;
 
 import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
 /**
@@ -49,4 +50,30 @@ public interface TimeLogger {
 	 */
 	public void logDrainDataCollectionTime(long time);
 
+	/**
+	 * Writes additional messages to compileTime OutputStreamWriter.
+	 * SNTimeInfoProcessor may use this method to log additional compilation
+	 * messages those are collected from {@link StreamNode}s.
+	 * 
+	 * @param msg
+	 */
+	public void logCompileTime(String msg);
+
+	/**
+	 * Writes additional messages to drainTime OutputStreamWriter.
+	 * SNTimeInfoProcessor may use this method to log additional draining
+	 * messages those are collected from {@link StreamNode}s.
+	 * 
+	 * @param msg
+	 */
+	public void logDrainTime(String msg);
+
+	/**
+	 * Writes additional messages to runTime OutputStreamWriter.
+	 * SNTimeInfoProcessor may use this method to log additional runTime
+	 * messages those are collected from {@link StreamNode}s.
+	 * 
+	 * @param msg
+	 */
+	public void logRunTime(String msg);
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 4c6a1e40..dfe0c83f 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -41,6 +41,18 @@ public void logDrainTime(long time) {
 		public void logDrainDataCollectionTime(long time) {
 		}
 
+		@Override
+		public void logCompileTime(String msg) {
+		}
+
+		@Override
+		public void logDrainTime(String msg) {
+		}
+
+		@Override
+		public void logRunTime(String msg) {
+		}
+
 	}
 
 	private static class TimeLoggerImpl implements TimeLogger {
@@ -118,6 +130,21 @@ public void logDrainDataCollectionTime(long time) {
 					String.format("Drain data collection time is %dms\n", time));
 		}
 
+		@Override
+		public void logCompileTime(String msg) {
+			write(compileTimeWriter, msg);
+		}
+
+		@Override
+		public void logDrainTime(String msg) {
+			write(drainTimeWriter, msg);
+		}
+
+		@Override
+		public void logRunTime(String msg) {
+			write(runTimeWriter, msg);
+		}
+
 	}
 
 	/**

From 4708755fda8ca3ada5795f51543481043819a834 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Nov 2014 17:36:45 +0800
Subject: [PATCH 415/881] Eclipse's sortMembers : TimeLogger.java

---
 .../mit/streamjit/impl/common/TimeLogger.java | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index 137178ec..b56ba8b6 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -14,13 +14,6 @@
  */
 public interface TimeLogger {
 
-	/**
-	 * This method shall be called to indicate the logger that a new
-	 * configuration has been received. Appropriate caller would be
-	 * {@link OnlineTuner}.
-	 */
-	public void newReconfiguration();
-
 	/**
 	 * Log the total compilation time of a new configuration. (Controller node
 	 * point of view).
@@ -30,18 +23,13 @@ public interface TimeLogger {
 	public void logCompileTime(long time);
 
 	/**
-	 * Log the time taken to generate fixed amount of steady state outputs.
-	 * 
-	 * @param time
-	 */
-	public void logRunTime(long time);
-
-	/**
-	 * Log total draining time.
+	 * Writes additional messages to compileTime OutputStreamWriter.
+	 * SNTimeInfoProcessor may use this method to log additional compilation
+	 * messages those are collected from {@link StreamNode}s.
 	 * 
-	 * @param time
+	 * @param msg
 	 */
-	public void logDrainTime(long time);
+	public void logCompileTime(String msg);
 
 	/**
 	 * Log total {@link DrainData} collection time.
@@ -51,13 +39,11 @@ public interface TimeLogger {
 	public void logDrainDataCollectionTime(long time);
 
 	/**
-	 * Writes additional messages to compileTime OutputStreamWriter.
-	 * SNTimeInfoProcessor may use this method to log additional compilation
-	 * messages those are collected from {@link StreamNode}s.
+	 * Log total draining time.
 	 * 
-	 * @param msg
+	 * @param time
 	 */
-	public void logCompileTime(String msg);
+	public void logDrainTime(long time);
 
 	/**
 	 * Writes additional messages to drainTime OutputStreamWriter.
@@ -68,6 +54,13 @@ public interface TimeLogger {
 	 */
 	public void logDrainTime(String msg);
 
+	/**
+	 * Log the time taken to generate fixed amount of steady state outputs.
+	 * 
+	 * @param time
+	 */
+	public void logRunTime(long time);
+
 	/**
 	 * Writes additional messages to runTime OutputStreamWriter.
 	 * SNTimeInfoProcessor may use this method to log additional runTime
@@ -76,4 +69,11 @@ public interface TimeLogger {
 	 * @param msg
 	 */
 	public void logRunTime(String msg);
+
+	/**
+	 * This method shall be called to indicate the logger that a new
+	 * configuration has been received. Appropriate caller would be
+	 * {@link OnlineTuner}.
+	 */
+	public void newReconfiguration();
 }

From 7e195416f82d384909568b51c143a0f1f4d66374 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Nov 2014 17:37:56 +0800
Subject: [PATCH 416/881] Eclipse's sort members : TimeLoggers.java

---
 .../impl/distributed/TimeLoggers.java         | 150 +++++++++---------
 1 file changed, 75 insertions(+), 75 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index dfe0c83f..e7352125 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -16,21 +16,40 @@
  */
 public class TimeLoggers {
 
+	public static class FileTimeLogger extends TimeLoggerImpl {
+
+		private static FileWriter getFileWriter(String name) {
+			FileWriter fw = null;
+			try {
+				fw = new FileWriter(name);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			return fw;
+		}
+
+		FileTimeLogger() {
+			super(getFileWriter("compileTime.txt"),
+					getFileWriter("runTime.txt"),
+					getFileWriter("drainTime.txt"));
+		}
+	}
+
 	/**
 	 * Logs nothing.
 	 */
 	public static class NoTimeLogger implements TimeLogger {
 
 		@Override
-		public void newReconfiguration() {
+		public void logCompileTime(long time) {
 		}
 
 		@Override
-		public void logCompileTime(long time) {
+		public void logCompileTime(String msg) {
 		}
 
 		@Override
-		public void logRunTime(long time) {
+		public void logDrainDataCollectionTime(long time) {
 		}
 
 		@Override
@@ -38,39 +57,43 @@ public void logDrainTime(long time) {
 		}
 
 		@Override
-		public void logDrainDataCollectionTime(long time) {
+		public void logDrainTime(String msg) {
 		}
 
 		@Override
-		public void logCompileTime(String msg) {
+		public void logRunTime(long time) {
 		}
 
 		@Override
-		public void logDrainTime(String msg) {
+		public void logRunTime(String msg) {
 		}
 
 		@Override
-		public void logRunTime(String msg) {
+		public void newReconfiguration() {
 		}
 
 	}
 
+	/**
+	 * Prints the values to the StdOut.
+	 * 
+	 */
+	public static class PrintTimeLogger extends TimeLoggerImpl {
+
+		public PrintTimeLogger() {
+			super(System.out, System.out, System.out);
+		}
+	}
+
 	private static class TimeLoggerImpl implements TimeLogger {
 
 		private final OutputStreamWriter compileTimeWriter;
 
-		private final OutputStreamWriter runTimeWriter;
-
 		private final OutputStreamWriter drainTimeWriter;
 
 		private int reconfigNo = 0;
 
-		TimeLoggerImpl(OutputStreamWriter compileW, OutputStreamWriter runW,
-				OutputStreamWriter drainW) {
-			compileTimeWriter = compileW;
-			runTimeWriter = runW;
-			drainTimeWriter = drainW;
-		}
+		private final OutputStreamWriter runTimeWriter;
 
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
 				OutputStream drainOs) {
@@ -79,32 +102,11 @@ private static class TimeLoggerImpl implements TimeLogger {
 			drainTimeWriter = getOSWriter(drainOs);
 		}
 
-		private OutputStreamWriter getOSWriter(OutputStream os) {
-			if (os == null)
-				return null;
-			return new OutputStreamWriter(os);
-		}
-
-		@Override
-		public void newReconfiguration() {
-			reconfigNo++;
-			String msg = String
-					.format("----------------------------%d----------------------------",
-							reconfigNo);
-			write(compileTimeWriter, msg);
-			write(runTimeWriter, msg);
-			write(drainTimeWriter, msg);
-		}
-
-		private void write(OutputStreamWriter osWriter, String msg) {
-			if (osWriter != null) {
-				try {
-					osWriter.write(msg);
-					osWriter.flush();
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			}
+		TimeLoggerImpl(OutputStreamWriter compileW, OutputStreamWriter runW,
+				OutputStreamWriter drainW) {
+			compileTimeWriter = compileW;
+			runTimeWriter = runW;
+			drainTimeWriter = drainW;
 		}
 
 		@Override
@@ -114,14 +116,8 @@ public void logCompileTime(long time) {
 		}
 
 		@Override
-		public void logRunTime(long time) {
-			write(runTimeWriter,
-					String.format("Execution time is %dms\n", time));
-		}
-
-		@Override
-		public void logDrainTime(long time) {
-			write(drainTimeWriter, String.format("Drain time is %dms\n", time));
+		public void logCompileTime(String msg) {
+			write(compileTimeWriter, msg);
 		}
 
 		@Override
@@ -131,8 +127,8 @@ public void logDrainDataCollectionTime(long time) {
 		}
 
 		@Override
-		public void logCompileTime(String msg) {
-			write(compileTimeWriter, msg);
+		public void logDrainTime(long time) {
+			write(drainTimeWriter, String.format("Drain time is %dms\n", time));
 		}
 
 		@Override
@@ -140,40 +136,44 @@ public void logDrainTime(String msg) {
 			write(drainTimeWriter, msg);
 		}
 
+		@Override
+		public void logRunTime(long time) {
+			write(runTimeWriter,
+					String.format("Execution time is %dms\n", time));
+		}
+
 		@Override
 		public void logRunTime(String msg) {
 			write(runTimeWriter, msg);
 		}
 
-	}
-
-	/**
-	 * Prints the values to the StdOut.
-	 * 
-	 */
-	public static class PrintTimeLogger extends TimeLoggerImpl {
-
-		public PrintTimeLogger() {
-			super(System.out, System.out, System.out);
+		@Override
+		public void newReconfiguration() {
+			reconfigNo++;
+			String msg = String
+					.format("----------------------------%d----------------------------",
+							reconfigNo);
+			write(compileTimeWriter, msg);
+			write(runTimeWriter, msg);
+			write(drainTimeWriter, msg);
 		}
-	}
-
-	public static class FileTimeLogger extends TimeLoggerImpl {
 
-		FileTimeLogger() {
-			super(getFileWriter("compileTime.txt"),
-					getFileWriter("runTime.txt"),
-					getFileWriter("drainTime.txt"));
+		private OutputStreamWriter getOSWriter(OutputStream os) {
+			if (os == null)
+				return null;
+			return new OutputStreamWriter(os);
 		}
 
-		private static FileWriter getFileWriter(String name) {
-			FileWriter fw = null;
-			try {
-				fw = new FileWriter(name);
-			} catch (IOException e) {
-				e.printStackTrace();
+		private void write(OutputStreamWriter osWriter, String msg) {
+			if (osWriter != null) {
+				try {
+					osWriter.write(msg);
+					osWriter.flush();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
 			}
-			return fw;
 		}
+
 	}
 }

From 4639baf0e7c17b9694688fb008d40f4061e08e44 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 24 Nov 2014 18:30:31 +0800
Subject: [PATCH 417/881] SNTimeInfoProcessorImpl uses TimeLogger

SNTimeInfoProcessorImpl uses TimeLogger to log all time measurement info
those are collected from StreamNodes.
---
 .../impl/distributed/StreamJitAppManager.java | 12 ++-
 .../impl/distributed/TimeLoggers.java         |  1 -
 .../common/SNTimeInfoProcessorImpl.java       | 74 ++++---------------
 .../distributed/runtimer/StreamNodeAgent.java |  4 +
 4 files changed, 28 insertions(+), 63 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 8d7b8a10..1a120054 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -39,6 +39,8 @@
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfoProcessorImpl;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -98,13 +100,17 @@ public class StreamJitAppManager {
 
 	private final TimeLogger logger;
 
+	private final SNTimeInfoProcessor timeInfoProcessor;
+
 	public StreamJitAppManager(Controller controller, StreamJitApp app,
-			ConfigurationManager cfgManager, ConnectionManager conManager, TimeLogger logger) {
+			ConfigurationManager cfgManager, ConnectionManager conManager,
+			TimeLogger logger) {
 		this.controller = controller;
 		this.app = app;
 		this.cfgManager = cfgManager;
 		this.conManager = conManager;
 		this.logger = logger;
+		this.timeInfoProcessor = new SNTimeInfoProcessorImpl(logger);
 		this.status = AppStatus.NOT_STARTED;
 		this.exP = new SNExceptionProcessorImpl();
 		this.ep = new ErrorProcessorImpl();
@@ -200,6 +206,10 @@ public SNExceptionProcessor exceptionProcessor() {
 		return exP;
 	}
 
+	public SNTimeInfoProcessor timeInfoProcessor() {
+		return timeInfoProcessor;
+	}
+
 	public long getFixedOutputTime() throws InterruptedException {
 		long time = tailChannel.getFixedOutputTime();
 		if (apStsPro.error) {
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index e7352125..6fc14ccd 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -174,6 +174,5 @@ private void write(OutputStreamWriter osWriter, String msg) {
 				}
 			}
 		}
-
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
index 6b3cdc21..e31c65d2 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
@@ -1,75 +1,27 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import java.io.FileWriter;
-import java.io.IOException;
-
-import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.CompilationTime;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
 
+/**
+ * Uses {@link TimeLogger} to log timing information.
+ * 
+ * @author sumanan
+ * @since Nov 24, 2014
+ */
 public class SNTimeInfoProcessorImpl implements SNTimeInfoProcessor {
 
-	private final FileWriter compileTimeWriter;
-
-	public SNTimeInfoProcessorImpl() {
-		compileTimeWriter = getFileWriter("CompileTime.txt");
-	}
-
-	private FileWriter getFileWriter(String name) {
-		FileWriter fw = null;
-		try {
-			fw = new FileWriter(name);
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		return fw;
-	}
-
-	/**
-	 * {@link StreamJitAppManager} can call this at every reconfiguration to
-	 * update time info log files.
-	 * 
-	 * @param reconfigNo
-	 */
-	public void reconfigNO(int reconfigNo) {
-		if (compileTimeWriter != null) {
-			try {
-				compileTimeWriter
-						.write(String
-								.format("-------------------------%d-------------------------\n",
-										reconfigNo));
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-	}
+	private final TimeLogger logger;
 
-	/**
-	 * The total compile time from {@link StreamJitAppManager} point.
-	 * 
-	 * @param time
-	 */
-	public void totalCompileTime(long time) {
-		if (compileTimeWriter != null) {
-			try {
-				compileTimeWriter.write(String.format(
-						"Total compile time %dms\n", time));
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
+	public SNTimeInfoProcessorImpl(TimeLogger logger) {
+		this.logger = logger;
 	}
 
 	@Override
 	public void process(CompilationTime compilationTime) {
-		if (compilationTime != null) {
-			String msg = String.format("Blob-%s-%.0fms\n",
-					compilationTime.blobID, compilationTime.milliSec);
-			try {
-				compileTimeWriter.write(msg);
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
+		String msg = String.format("Blob-%s-%.0fms\n", compilationTime.blobID,
+				compilationTime.milliSec);
+		logger.logCompileTime(msg);
 	}
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index bfd0ac29..017f7039 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -18,6 +18,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
 import edu.mit.streamjit.impl.distributed.common.SNMessageVisitor;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo.SystemInfoProcessor;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
@@ -291,6 +292,9 @@ public void visit(SNException snException) {
 
 		@Override
 		public void visit(SNTimeInfo timeInfo) {
+			assert manager != null : "StreamJitAppManager has not been set";
+			SNTimeInfoProcessor snTimeP = manager.timeInfoProcessor();
+			timeInfo.process(snTimeP);
 		}
 	}
 }
\ No newline at end of file

From c0ab122b6c87858cdc1bf97631d45bf06bff4229 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 09:21:01 +0800
Subject: [PATCH 418/881] Newline character added

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 6fc14ccd..b6138e50 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -151,7 +151,7 @@ public void logRunTime(String msg) {
 		public void newReconfiguration() {
 			reconfigNo++;
 			String msg = String
-					.format("----------------------------%d----------------------------",
+					.format("----------------------------%d----------------------------\n",
 							reconfigNo);
 			write(compileTimeWriter, msg);
 			write(runTimeWriter, msg);

From 2d9c7ba0ffe3cc8b1ce46b11a7e3dce47e6ebcd1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 09:36:02 +0800
Subject: [PATCH 419/881] FileTimeLogger takes prefix for the file names.

---
 .../DistributedStreamCompiler.java            |  2 +-
 .../impl/distributed/TimeLoggers.java         | 19 +++++++++++++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 98c76e6a..be857c72 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -132,7 +132,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		setConfiguration(controller, srcSink, stream, app, cfgManager,
 				conManager);
 
-		TimeLogger logger = new TimeLoggers.FileTimeLogger();
+		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
 				cfgManager, conManager, logger);
 		final AbstractDrainer drainer = new DistributedDrainer(manager);
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index b6138e50..636b22e0 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -16,6 +16,13 @@
  */
 public class TimeLoggers {
 
+	/**
+	 * Creates three files named prefix_compileTime.txt, prefix_runTime.txt and
+	 * prefix_drainTime.txt, and logs the time information.
+	 * 
+	 * @author sumanan
+	 * @since Nov 25, 2014
+	 */
 	public static class FileTimeLogger extends TimeLoggerImpl {
 
 		private static FileWriter getFileWriter(String name) {
@@ -28,10 +35,14 @@ private static FileWriter getFileWriter(String name) {
 			return fw;
 		}
 
-		FileTimeLogger() {
-			super(getFileWriter("compileTime.txt"),
-					getFileWriter("runTime.txt"),
-					getFileWriter("drainTime.txt"));
+		/**
+		 * @param prefix
+		 *            : prefix for the file names.
+		 */
+		public FileTimeLogger(String prefix) {
+			super(getFileWriter(String.format("%s_compileTime.txt", prefix)),
+					getFileWriter(String.format("%s_runTime.txt", prefix)),
+					getFileWriter(String.format("%s_drainTime.txt", prefix)));
 		}
 	}
 

From 37d7107cd1b5f20c35d2e9c24d3dd9ce7df34643 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 10:27:16 +0800
Subject: [PATCH 420/881] Refactored : sendCompilationTime() added.

---
 .../node/ConfigurationProcessorImpl.java        | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index c09c0cff..991f4b91 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -159,11 +159,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 					Stopwatch sw = Stopwatch.createStarted();
 					Blob b = bf.makeBlob(workerset, blobConfigs,
 							GlobalConstants.maxNumCores, drainData);
-					sw.stop();
-					CompilationTime ct = new CompilationTime(
-							Utils.getblobID(workerset),
-							sw.elapsed(TimeUnit.MILLISECONDS));
-					streamNode.controllerConnection.writeObject(ct);
+					sendCompilationTime(sw, Utils.getblobID(workerset));
 					blobSet.add(b);
 				} catch (Exception ex) {
 					ex.printStackTrace();
@@ -184,6 +180,17 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			return null;
 	}
 
+	private void sendCompilationTime(Stopwatch sw, Token blobID) {
+		sw.stop();
+		CompilationTime ct = new CompilationTime(blobID,
+				sw.elapsed(TimeUnit.MILLISECONDS));
+		try {
+			streamNode.controllerConnection.writeObject(ct);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
 	/**
 	 * Gets a Stream Graph from a jar file.
 	 * 

From 6a99c1c1b4a1b41ec43c0e0801dc7e2a18f60c2c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 11:53:23 +0800
Subject: [PATCH 421/881] New methods have been added to TimeLogger interface

Other classes may call the new methods to indicate the events and the TimeLogger
will manage the time measurements and the logging.
---
 .../mit/streamjit/impl/common/TimeLogger.java | 32 +++++++++++++++
 .../impl/distributed/TimeLoggers.java         | 40 ++++++++++++++++++-
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index b56ba8b6..9302c7c3 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -14,6 +14,38 @@
  */
 public interface TimeLogger {
 
+	/**
+	 * Compiler may call this method to indicate the compilation event has
+	 * started. TimeLogger may start a timer to measure the compilation time.
+	 */
+	public void compilationStarted();
+
+	/**
+	 * Compiler can call this method to indicate the compilation event has
+	 * finished. TimeLogger may stop the timer and log the compilation time.
+	 * 
+	 * @param isCompiled
+	 *            : Additional detail that goes with log.
+	 * @param msg
+	 *            : Additional details that go with log.
+	 */
+	public void compilationFinished(boolean isCompiled, String msg);
+
+	/**
+	 * Drainer or Tuner may call this method to indicate the draining even has
+	 * started. TimeLogger may start a timer to measure the compilation time.
+	 */
+	public void drainingStarted();
+
+	/**
+	 * Drainer or Tuner may call this method to indicate the draining even has
+	 * finished. TimeLogger may stop the timer and log the draining time.
+	 * 
+	 * @param msg
+	 *            : Additional details that go with log.
+	 */
+	public void drainingFinished(String msg);
+
 	/**
 	 * Log the total compilation time of a new configuration. (Controller node
 	 * point of view).
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 636b22e0..39f14572 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -51,6 +51,23 @@ public FileTimeLogger(String prefix) {
 	 */
 	public static class NoTimeLogger implements TimeLogger {
 
+		@Override
+		public void compilationFinished(boolean isCompiled, String msg) {
+		}
+
+		@Override
+		public void compilationStarted() {
+
+		}
+
+		@Override
+		public void drainingFinished(String msg) {
+		}
+
+		@Override
+		public void drainingStarted() {
+		}
+
 		@Override
 		public void logCompileTime(long time) {
 		}
@@ -82,7 +99,6 @@ public void logRunTime(String msg) {
 		@Override
 		public void newReconfiguration() {
 		}
-
 	}
 
 	/**
@@ -120,6 +136,28 @@ private static class TimeLoggerImpl implements TimeLogger {
 			drainTimeWriter = drainW;
 		}
 
+		@Override
+		public void compilationFinished(boolean isCompiled, String msg) {
+			// TODO Auto-generated method stub
+
+		}
+
+		@Override
+		public void compilationStarted() {
+		}
+
+		@Override
+		public void drainingFinished(String msg) {
+			// TODO Auto-generated method stub
+
+		}
+
+		@Override
+		public void drainingStarted() {
+			// TODO Auto-generated method stub
+
+		}
+
 		@Override
 		public void logCompileTime(long time) {
 			write(compileTimeWriter,

From 1066fb036e60342d80ae838c90708f68c6f8420a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 12:14:57 +0800
Subject: [PATCH 422/881] Implemented new TimeLogger methods.

---
 .../impl/distributed/TimeLoggers.java         | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 39f14572..faf805c8 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -4,6 +4,9 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.common.TimeLogger;
 
@@ -118,9 +121,13 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		private final OutputStreamWriter drainTimeWriter;
 
+		private final OutputStreamWriter runTimeWriter;
+
 		private int reconfigNo = 0;
 
-		private final OutputStreamWriter runTimeWriter;
+		private Stopwatch compileTimeSW = null;
+
+		private Stopwatch drainTimeSW = null;
 
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
 				OutputStream drainOs) {
@@ -138,24 +145,30 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		@Override
 		public void compilationFinished(boolean isCompiled, String msg) {
-			// TODO Auto-generated method stub
-
+			if (compileTimeSW != null) {
+				compileTimeSW.stop();
+				long time = compileTimeSW.elapsed(TimeUnit.MILLISECONDS);
+				logCompileTime(time);
+			}
 		}
 
 		@Override
 		public void compilationStarted() {
+			compileTimeSW = Stopwatch.createStarted();
 		}
 
 		@Override
 		public void drainingFinished(String msg) {
-			// TODO Auto-generated method stub
-
+			if (drainTimeSW != null) {
+				drainTimeSW.stop();
+				long time = drainTimeSW.elapsed(TimeUnit.MILLISECONDS);
+				logDrainTime(time);
+			}
 		}
 
 		@Override
 		public void drainingStarted() {
-			// TODO Auto-generated method stub
-
+			drainTimeSW = Stopwatch.createStarted();
 		}
 
 		@Override

From ba2cb7b30cbfb2dd39eae99d93c270287cdaf8cf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 12:18:32 +0800
Subject: [PATCH 423/881] Constructor calls the main constroctor with args.

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index faf805c8..200965b1 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -131,9 +131,8 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
 				OutputStream drainOs) {
-			compileTimeWriter = getOSWriter(compileOS);
-			runTimeWriter = getOSWriter(runOs);
-			drainTimeWriter = getOSWriter(drainOs);
+			this(getOSWriter(compileOS), getOSWriter(runOs),
+					getOSWriter(drainOs));
 		}
 
 		TimeLoggerImpl(OutputStreamWriter compileW, OutputStreamWriter runW,
@@ -220,7 +219,7 @@ public void newReconfiguration() {
 			write(drainTimeWriter, msg);
 		}
 
-		private OutputStreamWriter getOSWriter(OutputStream os) {
+		private static OutputStreamWriter getOSWriter(OutputStream os) {
 			if (os == null)
 				return null;
 			return new OutputStreamWriter(os);

From 5b49c8f08855768ed9c6a7b67b5c25ef540f093b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 12:22:45 +0800
Subject: [PATCH 424/881] StreamJitAppManager logs compilation time.

---
 src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 1a120054..585b8ae8 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -241,6 +241,7 @@ public boolean reconfigure(int multiplier) {
 
 		ImmutableMap<Integer, DrainData> drainDataMap = app.getDrainData();
 
+		logger.compilationStarted();
 		for (int nodeID : controller.getAllNodeIDs()) {
 			ConfigurationString json = new ConfigurationString(jsonStirng,
 					ConfigType.DYNAMIC, drainDataMap.get(nodeID));
@@ -250,6 +251,7 @@ public boolean reconfigure(int multiplier) {
 		setupHeadTail(conInfoMap, app.bufferMap, multiplier);
 
 		boolean isCompiled = apStsPro.waitForCompilation();
+		logger.compilationFinished(isCompiled, "");
 
 		if (isCompiled) {
 			start();

From c918f321911d7c0bf2ebc4f758185777c1de4f1f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 12:30:11 +0800
Subject: [PATCH 425/881] OnlineTuner logs draining time.

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 028d4e16..c134dab3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -188,15 +188,18 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 	 */
 	private boolean intermediateDraining() throws InterruptedException {
 		if (manager.isRunning()) {
+			logger.drainingStarted();
 			boolean state = drainer.startDraining(0);
 			if (!state) {
-				System.err
-						.println("Final drain has already been called. no more tuning.");
+				String msg = "Final drain has already been called. no more tuning.";
+				System.err.println(msg);
+				logger.drainingFinished(msg);
 				return false;
 			}
 
 			System.err.println("awaitDrainedIntrmdiate");
 			drainer.awaitDrainedIntrmdiate();
+			logger.drainingFinished("Intermediate");
 
 			drainer.awaitDrainData();
 			DrainData drainData = drainer.getDrainData();

From 8adb9731820954919e30c928e71dec6c63233231 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 14:48:29 +0800
Subject: [PATCH 426/881] new methods have been added to TimeLogger interface.

DrainDataCollection methods have been added to TimeLogger interface.
---
 .../mit/streamjit/impl/common/TimeLogger.java | 32 ++++++++++++++++---
 .../impl/distributed/TimeLoggers.java         | 25 +++++++++++++++
 2 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index 9302c7c3..bbf3ea93 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -6,8 +6,13 @@
 
 /**
  * Logs various time measurements for off line performance analysis. Controller
- * node can measure the time durations for different events and use this
- * interface to log those values.
+ * node may
+ * <ol>
+ * <li>Measure the time durations for different events and use this interface to
+ * log those values.
+ * <li>Call the appropriate event indicating methods and let the TimeLogger to
+ * measure and log the time values.
+ * </ol>
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Nov 22, 2014
@@ -32,13 +37,13 @@ public interface TimeLogger {
 	public void compilationFinished(boolean isCompiled, String msg);
 
 	/**
-	 * Drainer or Tuner may call this method to indicate the draining even has
-	 * started. TimeLogger may start a timer to measure the compilation time.
+	 * Drainer or Tuner may call this method to indicate the draining event has
+	 * started. TimeLogger may start a timer to measure the draining time.
 	 */
 	public void drainingStarted();
 
 	/**
-	 * Drainer or Tuner may call this method to indicate the draining even has
+	 * Drainer or Tuner may call this method to indicate the draining event has
 	 * finished. TimeLogger may stop the timer and log the draining time.
 	 * 
 	 * @param msg
@@ -46,6 +51,23 @@ public interface TimeLogger {
 	 */
 	public void drainingFinished(String msg);
 
+	/**
+	 * Drainer or Tuner may call this method to indicate the {@link DrainData}
+	 * collection event has started. TimeLogger may start a timer to measure the
+	 * drain data collection time.
+	 */
+	public void drainDataCollectionStarted();
+
+	/**
+	 * Drainer or Tuner may call this method to indicate the drain data
+	 * collection event has finished. TimeLogger may stop the timer and log the
+	 * drain data collection time.
+	 * 
+	 * @param msg
+	 *            : Additional details that go with log.
+	 */
+	public void drainDataCollectionFinished(String msg);
+
 	/**
 	 * Log the total compilation time of a new configuration. (Controller node
 	 * point of view).
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 200965b1..fcacdf93 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -102,6 +102,14 @@ public void logRunTime(String msg) {
 		@Override
 		public void newReconfiguration() {
 		}
+
+		@Override
+		public void drainDataCollectionStarted() {
+		}
+
+		@Override
+		public void drainDataCollectionFinished(String msg) {
+		}
 	}
 
 	/**
@@ -129,6 +137,8 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		private Stopwatch drainTimeSW = null;
 
+		private Stopwatch drainDataCollectionTimeSW = null;
+
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
 				OutputStream drainOs) {
 			this(getOSWriter(compileOS), getOSWriter(runOs),
@@ -170,6 +180,21 @@ public void drainingStarted() {
 			drainTimeSW = Stopwatch.createStarted();
 		}
 
+		@Override
+		public void drainDataCollectionStarted() {
+			drainDataCollectionTimeSW = Stopwatch.createStarted();
+		}
+
+		@Override
+		public void drainDataCollectionFinished(String msg) {
+			if (drainDataCollectionTimeSW != null) {
+				drainDataCollectionTimeSW.stop();
+				long time = drainDataCollectionTimeSW
+						.elapsed(TimeUnit.MILLISECONDS);
+				logDrainDataCollectionTime(time);
+			}
+		}
+
 		@Override
 		public void logCompileTime(long time) {
 			write(compileTimeWriter,

From 87d1076164ea9585dbc100229bf0f25f656e2c68 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 25 Nov 2014 14:52:22 +0800
Subject: [PATCH 427/881] OnlineTuner logs DrainDataCollection time.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index c134dab3..4bd1c2f3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -201,9 +201,11 @@ private boolean intermediateDraining() throws InterruptedException {
 			drainer.awaitDrainedIntrmdiate();
 			logger.drainingFinished("Intermediate");
 
+			logger.drainDataCollectionStarted();
 			drainer.awaitDrainData();
 			DrainData drainData = drainer.getDrainData();
 			app.drainData = drainData;
+			logger.drainDataCollectionFinished("");
 			return true;
 		} else
 			return true;

From 1f50bd22d1e3ecaf4d6629023a3d2791ade8d94b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 14:54:27 +0800
Subject: [PATCH 428/881] Method Rename:newReconfiguration->newConfiguration

---
 src/edu/mit/streamjit/impl/common/TimeLogger.java            | 2 +-
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java      | 5 +++--
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index bbf3ea93..997e3125 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -129,5 +129,5 @@ public interface TimeLogger {
 	 * configuration has been received. Appropriate caller would be
 	 * {@link OnlineTuner}.
 	 */
-	public void newReconfiguration();
+	public void newConfiguration();
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index fcacdf93..02a6f14c 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -100,7 +100,7 @@ public void logRunTime(String msg) {
 		}
 
 		@Override
-		public void newReconfiguration() {
+		public void newConfiguration() {
 		}
 
 		@Override
@@ -234,8 +234,9 @@ public void logRunTime(String msg) {
 		}
 
 		@Override
-		public void newReconfiguration() {
+		public void newConfiguration() {
 			reconfigNo++;
+
 			String msg = String
 					.format("----------------------------%d----------------------------\n",
 							reconfigNo);
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4bd1c2f3..4e5c5526 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -76,6 +76,7 @@ private void tune() {
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
+				logger.newConfiguration();
 				if (cfgJson == null) {
 					System.err.println("OpenTuner closed unexpectly.");
 					break;
@@ -158,7 +159,6 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			if (!intermediateDraining())
 				return new Pair<Boolean, Long>(false, -1l);
 
-			logger.newReconfiguration();
 			drainer.setBlobGraph(app.blobGraph);
 			int multiplier = getMultiplier(config);
 			if (manager.reconfigure(multiplier)) {

From c87480e0366990226cd03c8466828d576243baea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 14:59:15 +0800
Subject: [PATCH 429/881] updateTuningRoundTime() added

TimeLogger updates the time duration between two consecutive tunings.
---
 .../streamjit/impl/distributed/TimeLoggers.java | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 02a6f14c..fc3e50ef 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -139,6 +139,8 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		private Stopwatch drainDataCollectionTimeSW = null;
 
+		private Stopwatch tuningRoundSW = null;
+
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
 				OutputStream drainOs) {
 			this(getOSWriter(compileOS), getOSWriter(runOs),
@@ -235,6 +237,7 @@ public void logRunTime(String msg) {
 
 		@Override
 		public void newConfiguration() {
+			updateTuningRoundTime();
 			reconfigNo++;
 
 			String msg = String
@@ -245,6 +248,20 @@ public void newConfiguration() {
 			write(drainTimeWriter, msg);
 		}
 
+		private void updateTuningRoundTime() {
+			long time = 0;
+			if (tuningRoundSW == null)
+				tuningRoundSW = Stopwatch.createStarted();
+			else {
+				tuningRoundSW.stop();
+				time = tuningRoundSW.elapsed(TimeUnit.SECONDS);
+				tuningRoundSW.reset();
+				tuningRoundSW.start();
+			}
+			write(runTimeWriter,
+					String.format("Tuning round time - %dS\n", time));
+		}
+
 		private static OutputStreamWriter getOSWriter(OutputStream os) {
 			if (os == null)
 				return null;

From 832cc18120f1a4d4a2ce6630d50a5d461ef96825 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 15:24:28 +0800
Subject: [PATCH 430/881] Logs Draining time of each blob.

Each blobs sends the draining time to the controller and the controller
writes the time in a file.
---
 .../impl/distributed/common/SNTimeInfo.java   | 22 +++++++++++++++++++
 .../common/SNTimeInfoProcessorImpl.java       |  8 +++++++
 .../distributed/node/BlobsManagerImpl.java    | 13 +++++++++--
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
index 6da090f0..505c0b3c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfo.java
@@ -43,7 +43,29 @@ public void process(SNTimeInfoProcessor snTimeInfoProcessor) {
 
 	}
 
+	public static final class DrainingTime extends SNTimeInfo {
+
+		private static final long serialVersionUID = 1L;
+
+		public final Token blobID;
+
+		public final double milliSec;
+
+		public DrainingTime(Token blobID, double milliSec) {
+			this.blobID = blobID;
+			this.milliSec = milliSec;
+		}
+
+		@Override
+		public void process(SNTimeInfoProcessor snTimeInfoProcessor) {
+			snTimeInfoProcessor.process(this);
+		}
+
+	}
+
 	public interface SNTimeInfoProcessor {
 		public void process(CompilationTime compilationTime);
+
+		public void process(DrainingTime drainingTime);
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
index e31c65d2..94ff2a4a 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNTimeInfoProcessorImpl.java
@@ -2,6 +2,7 @@
 
 import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.CompilationTime;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.DrainingTime;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo.SNTimeInfoProcessor;
 
 /**
@@ -24,4 +25,11 @@ public void process(CompilationTime compilationTime) {
 				compilationTime.milliSec);
 		logger.logCompileTime(msg);
 	}
+
+	@Override
+	public void process(DrainingTime drainingTime) {
+		String msg = String.format("Blob-%s-%.0fms\n", drainingTime.blobID,
+				drainingTime.milliSec);
+		logger.logDrainTime(msg);
+	}
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 4aeb055d..c3c6fb49 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -44,6 +44,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.SNLocalBufferManager;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -716,7 +717,7 @@ public void process(DrainDataRequest drnDataReq) {
 		}
 	}
 
-	private static class DrainCallback implements Runnable {
+	private class DrainCallback implements Runnable {
 
 		private final BlobExecuter blobExec;
 
@@ -732,8 +733,16 @@ private static class DrainCallback implements Runnable {
 		@Override
 		public void run() {
 			sw.stop();
+			long time = sw.elapsed(TimeUnit.MILLISECONDS);
 			System.out.println("Time taken to drain " + blobExec.blobID
-					+ " is " + sw.elapsed(TimeUnit.MILLISECONDS) + " ms");
+					+ " is " + time + " ms");
+			try {
+				streamNode.controllerConnection
+						.writeObject(new SNTimeInfo.DrainingTime(
+								blobExec.blobID, time));
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 			blobExec.drained();
 		}
 	}

From d64cf11a2f687092cc3309a5cd305978b0aed35e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 16:37:21 +0800
Subject: [PATCH 431/881] BlockingInputChannel:Print msgs formatted.

Channel name is prefixed to all std out prints.
---
 .../node/BlockingInputChannel.java            | 24 +++++++------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 9eaaf711..f9d7d296 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -182,13 +182,9 @@ public void receiveData() {
 				if (stopType.get() > 1 && ++bufFullCount > 5) {
 					this.extraBuffer = new ExtraBuffer();
 					extraBuffer.write(obj);
-					System.err
-							.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
 					System.err
 							.println(name
 									+ " receiveData:Writing extra data in to extra buffer");
-					System.err
-							.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
 					break;
 				}
 			}
@@ -203,8 +199,8 @@ public void receiveData() {
 			softClosed = true;
 		} catch (EOFException e) {
 			// Other side is closed.
-			System.out
-					.println("receiveData:Closing by EOFExp. Not by softClose");
+			System.out.println(name
+					+ " receiveData:Closing by EOFExp. Not by softClose");
 			stopType.set(2);
 		} catch (IOException e) {
 			// TODO: Verify the program quality. Try to reconnect until it
@@ -273,13 +269,9 @@ private void finalReceive() {
 						this.extraBuffer = new ExtraBuffer();
 						extraBuffer.write(obj);
 						buffer = extraBuffer;
-						System.err
-								.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
 						System.err
 								.println(name
 										+ " finalReceive:Writing extra data in to extra buffer");
-						System.err
-								.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
 					}
 				}
 
@@ -297,7 +289,8 @@ private void finalReceive() {
 			} catch (IOException e) {
 				e.printStackTrace();
 				System.out
-						.println("finalReceive:Closing by IOException. Not by softClose.");
+						.println(name
+								+ " finalReceive:Closing by IOException. Not by softClose.");
 				hasData = false;
 			}
 		} while (hasData);
@@ -308,7 +301,7 @@ private void finalReceive() {
 	 * care about the data and just tuning a app for performance.
 	 */
 	private void discardAll() {
-		System.out.println("Discarding input data...");
+		System.out.println(name + " Discarding input data...");
 		boolean hasData;
 		do {
 			try {
@@ -322,7 +315,8 @@ private void discardAll() {
 				hasData = false;
 			} catch (IOException e) {
 				System.out
-						.println("finalReceive:Closing by IOException. Not by softClose.");
+						.println(name
+								+ " finalReceive:Closing by IOException. Not by softClose.");
 				hasData = false;
 			}
 		} while (hasData);
@@ -331,7 +325,7 @@ private void discardAll() {
 	private void reConnect() {
 		while (stopType.get() == 0) {
 			try {
-				System.out.println("TCPInputChannel : Reconnecting...");
+				System.out.println(name + " Reconnecting...");
 				this.connection.closeConnection();
 				connection = conProvider.getConnection(conInfo);
 				return;
@@ -350,7 +344,7 @@ public void stop(DrainType type) {
 		if (this.stopType.get() == 0) {
 			stopType.set(type.toint());
 		} else if (debugLevel > 0) {
-			System.err.println("Stop has already been called.");
+			System.err.println(name + " Stop has already been called.");
 		}
 	}
 

From a191b2cc522bdb4601da7bbcda7f4d10d798bd23 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 17:12:12 +0800
Subject: [PATCH 432/881] printMemoryStatus()'s print msg formatted.

---
 .../mit/streamjit/impl/distributed/common/Utils.java | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 69238644..0146118d 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -40,11 +40,13 @@ public static void printMemoryStatus() {
 		long heapMaxSize = Runtime.getRuntime().maxMemory();
 		long heapSize = Runtime.getRuntime().totalMemory();
 		long heapFreeSize = Runtime.getRuntime().freeMemory();
-		System.out.println("##############################################");
-		System.out.println("heapMaxSize = " + heapMaxSize / 1e6);
-		System.out.println("heapSize = " + heapSize / 1e6);
-		System.out.println("heapFreeSize = " + heapFreeSize / 1e6);
-		System.out.println("##############################################");
+		System.out.println("#########################");
+		System.out.println(String.format("heapMaxSize = %.2fMB",
+				heapMaxSize / 1e6));
+		System.out.println(String.format("heapSize = %.2fMB", heapSize / 1e6));
+		System.out.println(String.format("heapFreeSize = %.2fMB",
+				heapFreeSize / 1e6));
+		System.out.println("#########################");
 	}
 
 	public static void printOutOfMemory() {

From 4bcf54fb3040898a4b809b85ac919d6273aabdbf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 1 Dec 2014 17:15:01 +0800
Subject: [PATCH 433/881] Trivial screen prints have been removed.

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java | 4 ++--
 .../streamjit/impl/distributed/node/BlockingInputChannel.java | 4 ++--
 .../impl/distributed/node/ConfigurationProcessorImpl.java     | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index c3c6fb49..83484a30 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -734,8 +734,8 @@ private class DrainCallback implements Runnable {
 		public void run() {
 			sw.stop();
 			long time = sw.elapsed(TimeUnit.MILLISECONDS);
-			System.out.println("Time taken to drain " + blobExec.blobID
-					+ " is " + time + " ms");
+			// System.out.println("Time taken to drain " + blobExec.blobID +
+			// " is " + time + " ms");
 			try {
 				streamNode.controllerConnection
 						.writeObject(new SNTimeInfo.DrainingTime(
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index f9d7d296..a7945ffe 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -301,7 +301,7 @@ private void finalReceive() {
 	 * care about the data and just tuning a app for performance.
 	 */
 	private void discardAll() {
-		System.out.println(name + " Discarding input data...");
+		// System.out.println(name + " Discarding input data...");
 		boolean hasData;
 		do {
 			try {
@@ -316,7 +316,7 @@ private void discardAll() {
 			} catch (IOException e) {
 				System.out
 						.println(name
-								+ " finalReceive:Closing by IOException. Not by softClose.");
+								+ " discardAll:Closing by IOException. Not by softClose.");
 				hasData = false;
 			}
 		} while (hasData);
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 991f4b91..6e0e9a9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -82,7 +82,8 @@ private void processStaticCfg(String json) {
 	}
 
 	private void processDynamicCfg(String json, DrainData drainData) {
-		System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
+		System.out
+				.println("------------------------------------------------------------");
 		System.out.println("New Configuration.....");
 		releaseOldBM();
 		Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);

From 019451cb259bdc5836f5ea5ab9211e11c40333f6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 2 Dec 2014 15:06:56 +0800
Subject: [PATCH 434/881] Memory calculation corrected.MB=1024*1024.Not 1e6.

---
 .../mit/streamjit/impl/distributed/common/Utils.java | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 0146118d..a64365ae 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -40,12 +40,14 @@ public static void printMemoryStatus() {
 		long heapMaxSize = Runtime.getRuntime().maxMemory();
 		long heapSize = Runtime.getRuntime().totalMemory();
 		long heapFreeSize = Runtime.getRuntime().freeMemory();
+		int MEGABYTE = 1024 * 1024;
 		System.out.println("#########################");
-		System.out.println(String.format("heapMaxSize = %.2fMB",
-				heapMaxSize / 1e6));
-		System.out.println(String.format("heapSize = %.2fMB", heapSize / 1e6));
-		System.out.println(String.format("heapFreeSize = %.2fMB",
-				heapFreeSize / 1e6));
+		System.out.println(String.format("heapMaxSize = %dMB", heapMaxSize
+				/ MEGABYTE));
+		System.out.println(String
+				.format("heapSize = %dMB", heapSize / MEGABYTE));
+		System.out.println(String.format("heapFreeSize = %dMB", heapFreeSize
+				/ MEGABYTE));
 		System.out.println("#########################");
 	}
 

From caf3c6db40babd77b0bac13ac342d7b3a244648d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 5 Dec 2014 15:03:04 +0800
Subject: [PATCH 435/881] verifyTuningTimes() added.

This method just picks a few configurations and re-run the app to ensure the
time we reported to the opentuner is correct. This method can be called after
 the completion of the tuning.
---
 .../distributed/runtimer/OnlineTuner.java     | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4e5c5526..7a0b1e7a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -140,6 +140,26 @@ private void evaluate() {
 		terminate();
 	}
 
+	/**
+	 * This method just picks a few configurations and re-run the app to ensure
+	 * the time we reported to the opentuner is correct.
+	 * 
+	 * This method can be called after the completion of the tuning.
+	 */
+	private void verifyTuningTimes() {
+		int[] cfgNos = { 10, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500,
+				550, 600, 650, 700, 750, 800, 850, 900, 950, 1000 };
+		for (int n : cfgNos) {
+			String cfgName = String.format("%d%s.cfg", n, app.name);
+			Configuration cfg = ConfigurationUtils.readConfiguration(String
+					.format("configurations/%s/%s", app.name, cfgName));
+			if (cfg == null)
+				continue;
+			evaluateConfig(cfg, cfgName);
+		}
+		terminate();
+	}
+
 	/**
 	 * @param cfgJson
 	 * @param round

From 634aa14183a3af7ee9810532b41dd52439494519 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 5 Dec 2014 15:05:04 +0800
Subject: [PATCH 436/881] evaluateConfig() - Evaluation count reduced to 4.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 7a0b1e7a..ddcf7f21 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -290,7 +290,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
-		int count = 8;
+		int count = 4;
 		try {
 			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
 					true);

From 7f5b7fee0c40ebaca128b1a6467d780e4029787f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 5 Dec 2014 15:14:18 +0800
Subject: [PATCH 437/881] TimeLogProcessor has been added

Processes the Distributed StreamJit's time log files and generate a summary.
---
 .../mit/streamjit/util/TimeLogProcessor.java  | 155 ++++++++++++++++++
 1 file changed, 155 insertions(+)
 create mode 100644 src/edu/mit/streamjit/util/TimeLogProcessor.java

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
new file mode 100644
index 00000000..db68f359
--- /dev/null
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -0,0 +1,155 @@
+package edu.mit.streamjit.util;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Processes the Distributed StreamJit's time log files and generate summary.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Dec 5, 2014
+ */
+public class TimeLogProcessor {
+
+	public static void main(String[] args) throws IOException {
+		List<Integer> compileTime = processCompileTime();
+		List<Integer> runTime = processRunTime();
+		List<Integer> drainTime = processDrainTime();
+		List<Integer> sn1HeapMax = processSNHeap("st1.txt", true);
+		List<Integer> sn2HeapMax = processSNHeap("st2.txt", true);
+		List<Integer> sn1HeapSize = processSNHeap("st1.txt", false);
+		List<Integer> sn2HeapSize = processSNHeap("st2.txt", false);
+
+		FileWriter writer = new FileWriter("totalStats.txt");
+		int min = Integer.MAX_VALUE;
+
+		for (int i = 0; i < runTime.size(); i++) {
+			min = Math.min(min, runTime.get(i));
+			String msg = String.format(
+					"%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t\n",
+					i + 1, compileTime.get(i), runTime.get(i),
+					drainTime.get(i), sn1HeapMax.get(i), sn1HeapSize.get(i),
+					sn2HeapMax.get(i), sn2HeapSize.get(i), min);
+
+			writer.write(msg);
+		}
+		writer.close();
+
+		writeHeapStat("st1.txt");
+		writeHeapStat("st2.txt");
+	}
+
+	private static List<Integer> processCompileTime() throws IOException {
+		BufferedReader reader = new BufferedReader(new FileReader(
+				"FMRadioCore_compileTime.txt"));
+		FileWriter writer = new FileWriter("CompileTime.txt");
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("Total")) {
+				String[] arr = line.split(" ");
+				String time = arr[3].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				ret.add(val);
+				String data = String.format("%d\t%d\n", ++i, val);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
+	private static List<Integer> processRunTime() throws IOException {
+		BufferedReader reader = new BufferedReader(new FileReader(
+				"FMRadioCore_runTime.txt"));
+		FileWriter writer = new FileWriter("RunTime.txt");
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		int min = Integer.MAX_VALUE;
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("Execution")) {
+				String[] arr = line.split(" ");
+				String time = arr[3].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				min = Math.min(min, val);
+				ret.add(val);
+				String data = String
+						.format("%-6d\t%-6d\t%-6d\n", ++i, val, min);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
+	private static List<Integer> processDrainTime() throws IOException {
+		BufferedReader reader = new BufferedReader(new FileReader(
+				"FMRadioCore_drainTime.txt"));
+		FileWriter writer = new FileWriter("DrainTime.txt");
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("Drain time")) {
+				String[] arr = line.split(" ");
+				String time = arr[3].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				ret.add(val);
+				String data = String.format("%d\t%d\n", ++i, val);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
+	private static void writeHeapStat(String fileName) throws IOException {
+		List<Integer> heapSize = processSNHeap(fileName, false);
+		List<Integer> heapMaxSize = processSNHeap(fileName, true);
+		FileWriter writer = new FileWriter(String.format("%s_heapStatus.txt",
+				fileName));
+		for (int i = 0; i < heapSize.size(); i++) {
+			String msg = String.format("%-6d\t%-6d\t%-6d\n", i + 1,
+					heapSize.get(i), heapMaxSize.get(i));
+			writer.write(msg);
+		}
+		writer.close();
+	}
+
+	private static List<Integer> processSNHeap(String fileName,
+			Boolean isHeapMax) throws IOException {
+		String heapType = "heapSize";
+		if (isHeapMax)
+			heapType = "heapMaxSize";
+		BufferedReader reader = new BufferedReader(new FileReader(fileName));
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith(heapType)) {
+				String[] arr = line.split(" ");
+				String time = arr[2].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				ret.add(val);
+			}
+		}
+		reader.close();
+		return ret;
+	}
+}

From 99e7d76fb97e7f02fdc04c621ce24db77603aa09 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 5 Dec 2014 15:26:49 +0800
Subject: [PATCH 438/881] TimelogProcessor logs heap status separately.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index db68f359..819ea229 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -19,21 +19,14 @@ public static void main(String[] args) throws IOException {
 		List<Integer> compileTime = processCompileTime();
 		List<Integer> runTime = processRunTime();
 		List<Integer> drainTime = processDrainTime();
-		List<Integer> sn1HeapMax = processSNHeap("st1.txt", true);
-		List<Integer> sn2HeapMax = processSNHeap("st2.txt", true);
-		List<Integer> sn1HeapSize = processSNHeap("st1.txt", false);
-		List<Integer> sn2HeapSize = processSNHeap("st2.txt", false);
 
 		FileWriter writer = new FileWriter("totalStats.txt");
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 0; i < runTime.size(); i++) {
 			min = Math.min(min, runTime.get(i));
-			String msg = String.format(
-					"%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t\n",
-					i + 1, compileTime.get(i), runTime.get(i),
-					drainTime.get(i), sn1HeapMax.get(i), sn1HeapSize.get(i),
-					sn2HeapMax.get(i), sn2HeapSize.get(i), min);
+			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n", i + 1,
+					compileTime.get(i), runTime.get(i), drainTime.get(i), min);
 
 			writer.write(msg);
 		}

From 87ad69d99035e898d06c0a4a1a973b39f43fa9a2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 11 Dec 2014 17:26:29 +0800
Subject: [PATCH 439/881] BugFix: FinalReceive() writes an Object twice

Just after the creation of the extraBuffer(), finalReceive() writes the
received object twice in to the buffer. This bug is fixed.
---
 .../streamjit/impl/distributed/node/BlockingInputChannel.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index a7945ffe..7853dfc7 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -272,6 +272,7 @@ private void finalReceive() {
 						System.err
 								.println(name
 										+ " finalReceive:Writing extra data in to extra buffer");
+						break;
 					}
 				}
 

From b29148b5178118d556aa90de56d55f860e678254 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 15 Dec 2014 14:54:30 +0800
Subject: [PATCH 440/881] Renamed: DrainedData -> SNDrainedData

---
 .../impl/common/AbstractDrainer.java          | 34 ++++++++++---------
 .../impl/distributed/StreamJitAppManager.java |  6 ++--
 .../distributed/common/SNDrainElement.java    |  6 ++--
 .../distributed/node/BlobsManagerImpl.java    | 12 +++----
 4 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 09bb09e9..c4f66324 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -39,7 +39,7 @@
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
 /**
@@ -217,8 +217,8 @@ public final void awaitDrainData() throws InterruptedException {
 			drainDataLatch.await();
 	}
 
-	public final void newDrainData(DrainedData drainedData) {
-		blobGraph.getBlobNode(drainedData.blobID).setDrainData(drainedData);
+	public final void newSNDrainData(SNDrainedData snDrainedData) {
+		blobGraph.getBlobNode(snDrainedData.blobID).setDrainData(snDrainedData);
 		if (noOfDrainData.decrementAndGet() == 0) {
 			assert state == DrainerState.NODRAINING;
 			drainDataLatch.countDown();
@@ -239,22 +239,24 @@ public final DrainData getDrainData() {
 		Map<Token, ImmutableList<Object>> boundaryOutputData = new HashMap<>();
 
 		for (BlobNode node : blobGraph.blobNodes.values()) {
-			boundaryInputData.putAll(node.drainData.inputData);
-			boundaryOutputData.putAll(node.drainData.outputData);
+			boundaryInputData.putAll(node.snDrainData.inputData);
+			boundaryOutputData.putAll(node.snDrainData.outputData);
 			if (drainData == null)
-				drainData = node.drainData.drainData;
+				drainData = node.snDrainData.drainData;
 			else
-				drainData = drainData.merge(node.drainData.drainData);
+				drainData = drainData.merge(node.snDrainData.drainData);
 		}
 
 		ImmutableMap.Builder<Token, ImmutableList<Object>> dataBuilder = ImmutableMap
 				.builder();
 		for (Token t : Sets.union(boundaryInputData.keySet(),
 				boundaryOutputData.keySet())) {
-			ImmutableList<Object> in = boundaryInputData.get(t) != null ? boundaryInputData
-					.get(t) : ImmutableList.of();
-			ImmutableList<Object> out = boundaryOutputData.get(t) != null ? boundaryOutputData
-					.get(t) : ImmutableList.of();
+			ImmutableList<Object> in = boundaryInputData.get(t) != null
+					? boundaryInputData.get(t)
+					: ImmutableList.of();
+			ImmutableList<Object> out = boundaryOutputData.get(t) != null
+					? boundaryOutputData.get(t)
+					: ImmutableList.of();
 			dataBuilder.put(t, ImmutableList.builder().addAll(in).addAll(out)
 					.build());
 		}
@@ -575,7 +577,7 @@ public void setDrainer(AbstractDrainer drainer) {
 
 		public void clearDrainData() {
 			for (BlobNode node : blobNodes.values()) {
-				node.drainData = null;
+				node.snDrainData = null;
 			}
 		}
 
@@ -663,7 +665,7 @@ private static final class BlobNode {
 		/**
 		 * Intermediate drain data.
 		 */
-		private DrainedData drainData;
+		private SNDrainedData snDrainData;
 
 		private AbstractDrainer drainer;
 		/**
@@ -756,9 +758,9 @@ private void drain() {
 						TimeUnit.MILLISECONDS);
 		}
 
-		private void setDrainData(DrainedData drainedData) {
-			if (this.drainData == null) {
-				this.drainData = drainedData;
+		private void setDrainData(SNDrainedData drainedData) {
+			if (this.snDrainData == null) {
+				this.snDrainData = drainedData;
 				drainState.set(4);
 			} else
 				throw new AssertionError(
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 585b8ae8..cf044aed 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -34,7 +34,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.Drained;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
@@ -467,9 +467,9 @@ public void process(Drained drained) {
 		}
 
 		@Override
-		public void process(DrainedData drainedData) {
+		public void process(SNDrainedData snDrainedData) {
 			if (GlobalConstants.useDrainData)
-				drainer.newDrainData(drainedData);
+				drainer.newSNDrainData(snDrainedData);
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNDrainElement.java b/src/edu/mit/streamjit/impl/distributed/common/SNDrainElement.java
index dcb44336..62d27d51 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNDrainElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNDrainElement.java
@@ -58,7 +58,7 @@ public void process(SNDrainProcessor dp) {
 	 * the drain data of the blobs after the draining. See {@link DrainData} for
 	 * more information.
 	 */
-	public static final class DrainedData extends SNDrainElement {
+	public static final class SNDrainedData extends SNDrainElement {
 		private static final long serialVersionUID = 1L;
 
 		public final Token blobID;
@@ -66,7 +66,7 @@ public static final class DrainedData extends SNDrainElement {
 		public final ImmutableMap<Token, ImmutableList<Object>> inputData;
 		public final ImmutableMap<Token, ImmutableList<Object>> outputData;
 
-		public DrainedData(Token blobID, DrainData drainData,
+		public SNDrainedData(Token blobID, DrainData drainData,
 				ImmutableMap<Token, ImmutableList<Object>> inputData,
 				ImmutableMap<Token, ImmutableList<Object>> outputData) {
 			this.blobID = blobID;
@@ -92,6 +92,6 @@ public interface SNDrainProcessor {
 
 		public void process(Drained drained);
 
-		public void process(DrainedData drainedData);
+		public void process(SNDrainedData snDrainedData);
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 83484a30..a0b0719e 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -42,7 +42,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement.DrainedData;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
@@ -437,7 +437,7 @@ private void drained() {
 				if (crashed.get())
 					me = getEmptyDrainData();
 				else
-					me = getDrainData();
+					me = getSNDrainData();
 
 				try {
 					streamNode.controllerConnection.writeObject(me);
@@ -470,7 +470,7 @@ private void drained() {
 			// printDrainedStatus();
 		}
 
-		private DrainedData getDrainData() {
+		private SNDrainedData getSNDrainData() {
 			if (this.blob == null)
 				return getEmptyDrainData();
 
@@ -519,7 +519,7 @@ private DrainedData getDrainData() {
 				}
 			}
 
-			return new SNDrainElement.DrainedData(blobID, dd,
+			return new SNDrainElement.SNDrainedData(blobID, dd,
 					inputDataBuilder.build(), outputDataBuilder.build());
 		}
 
@@ -540,7 +540,7 @@ private void unprocessedDataFromLocalBuffer(
 			inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
 		}
 
-		private DrainedData getEmptyDrainData() {
+		private SNDrainedData getEmptyDrainData() {
 			drainState = 5;
 			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
 			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
@@ -550,7 +550,7 @@ private DrainedData getEmptyDrainData() {
 					.builder();
 			DrainData dd = new DrainData(dataBuilder.build(),
 					stateBuilder.build());
-			return new SNDrainElement.DrainedData(blobID, dd,
+			return new SNDrainElement.SNDrainedData(blobID, dd,
 					inputDataBuilder.build(), outputDataBuilder.build());
 		}
 

From aac79a3950afbeb9e52f50ffdfb20f2f8a9ebb99 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 16 Dec 2014 14:42:49 +0800
Subject: [PATCH 441/881] Ant sctipt to build and upload jar files to Lanka

This Ant script makes jar file of a StreamJit app and a StreamNode.
sendtoLanka target in this Ant script will jar the StreamJit app and the
treamNode and send it to Lanka node.
---
 jarapp.properties |  5 ++++
 jarapp.xml        | 66 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 jarapp.properties
 create mode 100644 jarapp.xml

diff --git a/jarapp.properties b/jarapp.properties
new file mode 100644
index 00000000..e123880a
--- /dev/null
+++ b/jarapp.properties
@@ -0,0 +1,5 @@
+app=NestedSplitJoin
+mainclass=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
+username=sumanan
+##password=abc123
+
diff --git a/jarapp.xml b/jarapp.xml
new file mode 100644
index 00000000..f3652820
--- /dev/null
+++ b/jarapp.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--Sumanan, sumanan@mit.edu, 15-12-2014 -->
+<project default="buildApp" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
+	<property file="jarapp.properties" />
+	<!--property name="app" value="NestedSplitJoin" /-->
+	<!--property name="mainclass" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->
+	<!-- If you want, you can hard code your username and the password as plain 
+		text here. Comment these properties (username or password or both) if you 
+		don't want to keep it as a plain text. If these properties are not set (commented), 
+		Ant will ask you to enter those at runtime. -->
+	<!-- property name="username" value="sumanan" /-->
+	<!-- property name="password" value="" / -->
+
+	<target name="buildApp" depends="StreamNode" description="Exports ${app}.jar">
+		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
+			<manifest>
+				<attribute name="Main-Class" value="${mainclass}" />
+				<attribute name="Class-Path" value="." />
+			</manifest>
+			<fileset dir="bin" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/jsonp/javax.json-1.0-fab.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ASM/asm-debug-all-4.1.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ServiceProviderProcessor/ServiceProviderProcessor.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/JOptSimple/jopt-simple-4.5.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/sqlite/sqlite-jdbc-3.7.2.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/Guava/guava-15.0.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar" />
+		</jar>
+	</target>
+
+	<target name="StreamNode" description="Exports StreamNode.jar">
+		<jar destfile="StreamNode.jar" filesetmanifest="mergewithoutmain">
+			<manifest>
+				<attribute name="Main-Class" value="edu.mit.streamjit.impl.distributed.node.StreamNode" />
+				<attribute name="Class-Path" value="." />
+			</manifest>
+			<fileset dir="bin" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/CopyLibs/org-netbeans-modules-java-j2seproject-copylibstask.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/jsonp/javax.json-1.0-fab.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ASM/asm-debug-all-4.1.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/ServiceProviderProcessor/ServiceProviderProcessor.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/JOptSimple/jopt-simple-4.5.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/sqlite/sqlite-jdbc-3.7.2.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/Guava/guava-15.0.jar" />
+			<zipfileset excludes="META-INF/*.SF" src="lib/BridJ/bridj-0.7-20140122.002307-56-c-only.jar" />
+		</jar>
+	</target>
+
+	<target name="sendtoLanka" depends="buildApp, getLogin">
+		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
+			<fileset file="StreamNode.jar" />
+			<fileset file="${app}.jar" />
+		</scp>
+	</target>
+
+	<target name="getLogin">
+		<input message="Enter Username:" addproperty="username" />
+		<input message="Enter Password:" addproperty="password">
+			<handler classname="org.apache.tools.ant.input.SecureInputHandler" />
+		</input>
+		<!--echo message="Username: ${username}" /-->
+		<!--echo message="Password: ${password}" /-->
+	</target>
+
+</project>

From a55d59e5fa4925878242c552204ca83825cb897c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 16 Dec 2014 17:52:32 +0800
Subject: [PATCH 442/881] Ant shexec. Not completed.

---
 jarapp.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index f3652820..7f65fe5a 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -63,4 +63,8 @@
 		<!--echo message="Password: ${password}" /-->
 	</target>
 
+	<target name="remotesetup" depends="getLogin">
+		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/helloworld.sh" input="kkkk" />
+	</target>
+
 </project>

From 62609876455013ca328d46fa69cd03d50facc5a0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 18 Dec 2014 10:45:47 +0800
Subject: [PATCH 443/881] Prints DrainData Statistics

Each StreamNode prints the size of the drain data. The corresponding print lines
are commented in this commit. But those lines can be uncommented when debugging.
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index a0b0719e..021d602a 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -494,8 +494,7 @@ private SNDrainedData getSNDrainData() {
 					BoundaryChannel chanl = inputChannels.get(t);
 					ImmutableList<Object> draindata = chanl
 							.getUnprocessedData();
-					// System.out.println(String.format(
-					// "No of unprocessed data of %s is %d",
+					// System.out.println(String.format("From %s - %d",
 					// chanl.name(), draindata.size()));
 					inputDataBuilder.put(t, draindata);
 				}
@@ -512,8 +511,7 @@ private SNDrainedData getSNDrainData() {
 					BoundaryChannel chanl = outputChannels.get(t);
 					ImmutableList<Object> draindata = chanl
 							.getUnprocessedData();
-					// System.out.println(String.format(
-					// "No of unprocessed data of %s is %d",
+					// System.out.println(String.format("From %s - %d",
 					// chanl.name(), draindata.size()));
 					outputDataBuilder.put(t, draindata);
 				}
@@ -537,6 +535,8 @@ private void unprocessedDataFromLocalBuffer(
 			} else {
 				bufArray = bufferCleaner.copiedBuffer(t);
 			}
+			// System.out.println(String.format("From LocalBuffer: %s - %d", t,
+			// bufArray.length));
 			inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
 		}
 

From c0d6746fc8c9eb39f65df8b0e276b12d791c06fb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 18 Dec 2014 12:21:04 +0800
Subject: [PATCH 444/881] AbstractDrainer: Comments added.

---
 .../mit/streamjit/impl/common/AbstractDrainer.java   | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index c4f66324..7ec7a7a9 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -252,11 +252,9 @@ public final DrainData getDrainData() {
 		for (Token t : Sets.union(boundaryInputData.keySet(),
 				boundaryOutputData.keySet())) {
 			ImmutableList<Object> in = boundaryInputData.get(t) != null
-					? boundaryInputData.get(t)
-					: ImmutableList.of();
+					? boundaryInputData.get(t) : ImmutableList.of();
 			ImmutableList<Object> out = boundaryOutputData.get(t) != null
-					? boundaryOutputData.get(t)
-					: ImmutableList.of();
+					? boundaryOutputData.get(t) : ImmutableList.of();
 			dataBuilder.put(t, ImmutableList.builder().addAll(in).addAll(out)
 					.build());
 		}
@@ -344,9 +342,9 @@ public final void awaitDrained() throws InterruptedException {
 	public final void awaitDrainedIntrmdiate() throws InterruptedException {
 		intermediateLatch.await();
 
-		// Just for debugging purpose. To make effect of this code snippet
-		// comment the above, intermediateLatch.await(), line. Otherwise no
-		// effect.
+		// The following while loop is added just for debugging purpose. To
+		// activate the following while loop code snippet, comment the above
+		// [intermediateLatch.await()] line.
 		while (intermediateLatch.getCount() != 0) {
 			Thread.sleep(3000);
 			System.out.println("****************************************");

From ad991aa89baa26601114d29cfb046fefd7d0984a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Dec 2014 22:02:38 +0800
Subject: [PATCH 445/881] Bug Fix:(92). Unbalanced DrainData.

Sometimes unbalanced amount of DranData comes when reconfiguring. The problem
was,We copy the draindata to both input edge and output edge of the blobs,
if both up blob and down blob are happen to fall on a same SN.
---
 .../impl/distributed/node/ConfigurationProcessorImpl.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 6e0e9a9d..805d3faa 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -158,8 +158,10 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 				try {
 					BlobFactory bf = bs.getBlobFactory();
 					Stopwatch sw = Stopwatch.createStarted();
+					DrainData dd = drainData == null ? null : drainData
+							.subset(workIdentifiers);
 					Blob b = bf.makeBlob(workerset, blobConfigs,
-							GlobalConstants.maxNumCores, drainData);
+							GlobalConstants.maxNumCores, dd);
 					sendCompilationTime(sw, Utils.getblobID(workerset));
 					blobSet.add(b);
 				} catch (Exception ex) {

From bc33e935391ddda084329612d38682a5d730c34f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Dec 2014 22:52:04 +0800
Subject: [PATCH 446/881] A method minimizeDrainData() has been added.

Uses an Interpreter blob to clear or minimize a DrainData. This method can be
called after a final draining to clear the data in the intermediate buffers.
---
 .../impl/distributed/StreamJitApp.java        | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index fa92f629..0f600620 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -5,12 +5,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.AbstractReadOnlyBuffer;
+import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.blob.Buffer;
@@ -22,6 +25,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.impl.interp.Interpreter;
 
 /**
  * This class contains all information about the current streamJit application
@@ -183,6 +187,64 @@ public ImmutableMap<Integer, DrainData> getDrainData() {
 		return builder.build();
 	}
 
+	/**
+	 * Uses an {@link Interpreter} blob to clear or minimize a {@link DrainData}
+	 * . This method can be called after a final draining to clear the data in
+	 * the intermediate buffers.
+	 * 
+	 * @param drainData
+	 *            : {@link DrainData} that is received after a draining.
+	 * @return : A {@link DrainData} that remains after running an
+	 *         {@link Interpreter} blob.
+	 */
+	public DrainData minimizeDrainData(DrainData drainData) {
+		Interpreter.InterpreterBlobFactory interpFactory = new Interpreter.InterpreterBlobFactory();
+		Blob interp = interpFactory.makeBlob(Workers
+				.getAllWorkersInGraph(source), interpFactory
+				.getDefaultConfiguration(Workers.getAllWorkersInGraph(source)),
+				1, drainData);
+		interp.installBuffers(bufferMapWithEmptyHead());
+		Runnable interpCode = interp.getCoreCode(0);
+		final AtomicBoolean interpFinished = new AtomicBoolean();
+		interp.drain(new Runnable() {
+			@Override
+			public void run() {
+				interpFinished.set(true);
+			}
+		});
+		while (!interpFinished.get())
+			interpCode.run();
+		return interp.getDrainData();
+	}
+
+	/**
+	 * Remove the original headbuffer and replace it with a new empty buffer.
+	 */
+	private ImmutableMap<Token, Buffer> bufferMapWithEmptyHead() {
+		ImmutableMap.Builder<Token, Buffer> bufMapBuilder = ImmutableMap
+				.builder();
+		Buffer head = new AbstractReadOnlyBuffer() {
+			@Override
+			public int size() {
+				return 0;
+			}
+
+			@Override
+			public Object read() {
+				return null;
+			}
+		};
+
+		Token headToken = Token.createOverallInputToken(source);
+		for (Map.Entry<Token, Buffer> en : bufferMap.entrySet()) {
+			if (en.getKey().equals(headToken))
+				bufMapBuilder.put(headToken, head);
+			else
+				bufMapBuilder.put(en);
+		}
+		return bufMapBuilder.build();
+	}
+
 	private Set<Integer> getWorkerIds(List<Set<Worker<?, ?>>> blobList) {
 		Set<Integer> workerIds = new HashSet<>();
 		for (Set<Worker<?, ?>> blobworkers : blobList) {

From 2cf1274e875afffe61995948760671ab369d864b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Dec 2014 23:12:09 +0800
Subject: [PATCH 447/881] Print msgs removed from updateDrainDataStatistics

Print msgs have been removed from AbstractDrainer.updateDrainDataStatistics().
---
 .../streamjit/impl/common/AbstractDrainer.java  | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 7ec7a7a9..7a55cb85 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -262,29 +262,22 @@ public final DrainData getDrainData() {
 		ImmutableTable<Integer, String, Object> state = ImmutableTable.of();
 		DrainData draindata1 = new DrainData(dataBuilder.build(), state);
 		drainData = drainData.merge(draindata1);
-		updateDrainDataStatistics(drainData, false);
+		updateDrainDataStatistics(drainData);
 		// dumpDrainData(drainData);
 		return drainData;
 	}
 
-	private void updateDrainDataStatistics(DrainData drainData, boolean print) {
+	private void updateDrainDataStatistics(DrainData drainData) {
 		if (drainDataStatistics == null) {
 			drainDataStatistics = new HashMap<>();
 			for (Token t : drainData.getData().keySet()) {
 				drainDataStatistics.put(t, new ArrayList<Integer>());
 			}
 		}
-
 		for (Token t : drainData.getData().keySet()) {
-			drainDataStatistics.get(t).add(drainData.getData().get(t).size());
-			if (print) {
-				System.out.print("Aggregated data: " + t.toString() + " - "
-						+ drainData.getData().get(t).size() + " - ");
-				// for (Object o : drainData.getData().get(t)) {
-				// System.out.print(o.toString() + ", ");
-				// }
-				System.out.print('\n');
-			}
+			int size = drainData.getData().get(t).size();
+			drainDataStatistics.get(t).add(size);
+
 		}
 	}
 

From d6865f4b54c815151dc340165bbc9ccd646dba72 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 19 Dec 2014 23:16:25 +0800
Subject: [PATCH 448/881] AbstractDrainer.printDrainDataStats() added.

---
 .../impl/common/AbstractDrainer.java          | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 7a55cb85..24184c0c 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -263,6 +263,7 @@ public final DrainData getDrainData() {
 		DrainData draindata1 = new DrainData(dataBuilder.build(), state);
 		drainData = drainData.merge(draindata1);
 		updateDrainDataStatistics(drainData);
+		// printDrainDataStats(drainData);
 		// dumpDrainData(drainData);
 		return drainData;
 	}
@@ -281,6 +282,27 @@ private void updateDrainDataStatistics(DrainData drainData) {
 		}
 	}
 
+	private void printDrainDataStats(DrainData drainData) {
+		try {
+			FileWriter writer = new FileWriter("draindatasize.txt", true);
+			writer.write("-----------------------------------------------------------\n");
+			System.out.println("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^");
+
+			for (Token t : drainData.getData().keySet()) {
+				int size = drainData.getData().get(t).size();
+				if (size != 0) {
+					String msg = String.format("%s - %d\n", t.toString(), size);
+					System.out.print(msg);
+					writer.write(msg);
+				}
+			}
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
 	private void dumpDrainData(DrainData drainData) {
 		try {
 			FileOutputStream fout = new FileOutputStream("DrainData");

From 8d64775ccceb24086d4377a1e7cbeb051d87022c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 11:48:34 +0800
Subject: [PATCH 449/881] AbstractDrainer handles DrainData

AbstractDrainer.awaitDrainData() awaits for DrainData from all StreamNodes,
combines the all received DrainData and set the combined DrainData to
StreamJitApp.drainData.
---
 .../impl/common/AbstractDrainer.java          | 20 ++++++++++++++++---
 .../distributed/runtimer/OnlineTuner.java     |  3 ---
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 24184c0c..d18e3f30 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -37,9 +37,11 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
 /**
@@ -113,9 +115,12 @@ public abstract class AbstractDrainer {
 	 */
 	private DrainerState state;
 
-	public AbstractDrainer() {
+	private final StreamJitApp app;
+
+	public AbstractDrainer(StreamJitApp app) {
 		state = DrainerState.NODRAINING;
 		finalLatch = new CountDownLatch(1);
+		this.app = app;
 	}
 
 	/**
@@ -212,9 +217,18 @@ public final void drained(Token blobID) {
 		blobGraph.getBlobNode(blobID).drained();
 	}
 
+	/**
+	 * Awaits for {@link DrainData} from all {@link StreamNode}s, combines the
+	 * all received DrainData and set the combined DrainData to
+	 * {@link StreamJitApp#drainData}.
+	 * 
+	 * @throws InterruptedException
+	 */
 	public final void awaitDrainData() throws InterruptedException {
-		if (GlobalConstants.useDrainData)
+		if (GlobalConstants.useDrainData) {
 			drainDataLatch.await();
+			app.drainData = getDrainData();
+		}
 	}
 
 	public final void newSNDrainData(SNDrainedData snDrainedData) {
@@ -231,7 +245,7 @@ public final void newSNDrainData(SNDrainedData snDrainedData) {
 	/**
 	 * @return Aggregated DrainData after the draining.
 	 */
-	public final DrainData getDrainData() {
+	private final DrainData getDrainData() {
 		if (!GlobalConstants.useDrainData)
 			return null;
 		DrainData drainData = null;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ddcf7f21..49ed5659 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -4,7 +4,6 @@
 import java.io.FileWriter;
 import java.io.IOException;
 
-import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -223,8 +222,6 @@ private boolean intermediateDraining() throws InterruptedException {
 
 			logger.drainDataCollectionStarted();
 			drainer.awaitDrainData();
-			DrainData drainData = drainer.getDrainData();
-			app.drainData = drainData;
 			logger.drainDataCollectionFinished("");
 			return true;
 		} else

From 0cca97cde2ae39283ba49fb1e1308a7074b0b17a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 13:08:36 +0800
Subject: [PATCH 450/881] drainFinal() and drainIntermediate() added.

Two new methods have been added to AbstractDrainer. These methods will simplyfy
the drain calls.
---
 .../impl/common/AbstractDrainer.java          | 61 ++++++++++++++++---
 .../impl/distributed/StreamJitAppManager.java |  2 +-
 .../distributed/runtimer/OnlineTuner.java     | 18 +-----
 3 files changed, 56 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index d18e3f30..9a970e51 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -115,12 +115,15 @@ public abstract class AbstractDrainer {
 	 */
 	private DrainerState state;
 
+	private final TimeLogger logger;
+
 	private final StreamJitApp app;
 
-	public AbstractDrainer(StreamJitApp app) {
+	public AbstractDrainer(StreamJitApp app, TimeLogger logger) {
 		state = DrainerState.NODRAINING;
 		finalLatch = new CountDownLatch(1);
 		this.app = app;
+		this.logger = logger;
 	}
 
 	/**
@@ -169,10 +172,7 @@ public final boolean startDraining(int type) {
 			boolean isFinal = false;
 			switch (type) {
 				case 0 :
-					this.blobGraph.clearDrainData();
 					this.state = DrainerState.INTERMEDIATE;
-					drainDataLatch = new CountDownLatch(1);
-					intermediateLatch = new CountDownLatch(1);
 					break;
 				case 1 :
 					this.state = DrainerState.FINAL;
@@ -186,6 +186,10 @@ public final boolean startDraining(int type) {
 							"Invalid draining type. type can be 0, 1, or 2.");
 			}
 
+			this.blobGraph.clearDrainData();
+			drainDataLatch = new CountDownLatch(1);
+			intermediateLatch = new CountDownLatch(1);
+
 			try {
 				prepareDraining(isFinal);
 			} catch (Exception e) {
@@ -209,6 +213,51 @@ public final boolean startDraining(int type) {
 		}
 	}
 
+	public boolean drainIntermediate() throws InterruptedException {
+		logger.drainingStarted();
+		boolean state = startDraining(0);
+		if (!state) {
+			String msg = "Final drain has already been called. no more tuning.";
+			System.err.println(msg);
+			logger.drainingFinished(msg);
+			return false;
+		}
+
+		System.err.println("awaitDrainedIntrmdiate");
+		awaitDrainedIntrmdiate();
+		drainingDone(this.state == DrainerState.FINAL);
+		logger.drainingFinished("Intermediate");
+		logger.drainDataCollectionStarted();
+		awaitDrainData();
+		logger.drainDataCollectionFinished("");
+		return true;
+	}
+
+	public boolean drainFinal(Boolean isSemeFinal) throws InterruptedException {
+		int drainType = 2;
+		if (isSemeFinal)
+			drainType = 1;
+		logger.drainingStarted();
+		boolean state = startDraining(drainType);
+		if (!state) {
+			String msg = "Final drain has already been called.";
+			System.err.println(msg);
+			logger.drainingFinished(msg);
+			return false;
+		}
+
+		System.err.println("awaitDrainedIntrmdiate");
+		awaitDrainedIntrmdiate();
+		drainingDone(false);
+		logger.drainingFinished("Intermediate");
+		logger.drainDataCollectionStarted();
+		awaitDrainData();
+		logger.drainDataCollectionFinished("");
+		app.drainData = app.minimizeDrainData(app.drainData);
+		drainingDone(true);
+		stop();
+		return true;
+	}
 	/**
 	 * Once draining of a blob is done, it has to inform to the drainer by
 	 * calling this method.
@@ -476,12 +525,10 @@ private void drainingDone(BlobNode blobNode) {
 		assert state != DrainerState.NODRAINING : "Illegal call. Drainer is not in draining mode.";
 		drainingDone(blobNode.blobID, state == DrainerState.FINAL);
 		if (unDrainedNodes.decrementAndGet() == 0) {
-			drainingDone(state == DrainerState.FINAL);
+			intermediateLatch.countDown();
 			if (state == DrainerState.FINAL) {
-				finalLatch.countDown();
 			} else {
 				state = DrainerState.NODRAINING;
-				intermediateLatch.countDown();
 			}
 
 			if (GlobalConstants.needDrainDeadlockHandler)
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index cf044aed..3e8c9e03 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -173,7 +173,7 @@ public void drainingFinished(boolean isFinal) {
 		isRunning = false;
 
 		Stopwatch sw = stopwatchRef.get();
-		if (sw != null) {
+		if (sw != null && sw.isRunning()) {
 			sw.stop();
 			long time = sw.elapsed(TimeUnit.MILLISECONDS);
 			System.out.println("Draining time is " + time + " milli seconds");
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 49ed5659..15632495 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -207,23 +207,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 	 */
 	private boolean intermediateDraining() throws InterruptedException {
 		if (manager.isRunning()) {
-			logger.drainingStarted();
-			boolean state = drainer.startDraining(0);
-			if (!state) {
-				String msg = "Final drain has already been called. no more tuning.";
-				System.err.println(msg);
-				logger.drainingFinished(msg);
-				return false;
-			}
-
-			System.err.println("awaitDrainedIntrmdiate");
-			drainer.awaitDrainedIntrmdiate();
-			logger.drainingFinished("Intermediate");
-
-			logger.drainDataCollectionStarted();
-			drainer.awaitDrainData();
-			logger.drainDataCollectionFinished("");
-			return true;
+			return drainer.drainIntermediate();
 		} else
 			return true;
 	}

From 103d8d8be22de2ab8aa414acc82a5d1da153dfbc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 21:11:46 +0800
Subject: [PATCH 451/881] Stop throwing InterruptedException

---
 .../impl/common/AbstractDrainer.java          | 29 +++++++++++++++----
 .../DistributedStreamCompiler.java            |  3 +-
 .../runtimer/DistributedDrainer.java          |  8 +++--
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 9a970e51..58c85be9 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -213,7 +213,7 @@ public final boolean startDraining(int type) {
 		}
 	}
 
-	public boolean drainIntermediate() throws InterruptedException {
+	public boolean drainIntermediate() {
 		logger.drainingStarted();
 		boolean state = startDraining(0);
 		if (!state) {
@@ -224,16 +224,24 @@ public boolean drainIntermediate() throws InterruptedException {
 		}
 
 		System.err.println("awaitDrainedIntrmdiate");
-		awaitDrainedIntrmdiate();
+		try {
+			awaitDrainedIntrmdiate();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
 		drainingDone(this.state == DrainerState.FINAL);
 		logger.drainingFinished("Intermediate");
 		logger.drainDataCollectionStarted();
-		awaitDrainData();
+		try {
+			awaitDrainData();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
 		logger.drainDataCollectionFinished("");
 		return true;
 	}
 
-	public boolean drainFinal(Boolean isSemeFinal) throws InterruptedException {
+	public boolean drainFinal(Boolean isSemeFinal) {
 		int drainType = 2;
 		if (isSemeFinal)
 			drainType = 1;
@@ -247,17 +255,26 @@ public boolean drainFinal(Boolean isSemeFinal) throws InterruptedException {
 		}
 
 		System.err.println("awaitDrainedIntrmdiate");
-		awaitDrainedIntrmdiate();
+		try {
+			awaitDrainedIntrmdiate();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
 		drainingDone(false);
 		logger.drainingFinished("Intermediate");
 		logger.drainDataCollectionStarted();
-		awaitDrainData();
+		try {
+			awaitDrainData();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
 		logger.drainDataCollectionFinished("");
 		app.drainData = app.minimizeDrainData(app.drainData);
 		drainingDone(true);
 		stop();
 		return true;
 	}
+
 	/**
 	 * Once draining of a blob is done, it has to inform to the drainer by
 	 * calling this method.
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index be857c72..eebb8223 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -135,7 +135,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
 				cfgManager, conManager, logger);
-		final AbstractDrainer drainer = new DistributedDrainer(manager);
+		final AbstractDrainer drainer = new DistributedDrainer(app, logger,
+				manager);
 		drainer.setBlobGraph(app.blobGraph);
 
 		boolean needTermination = setBufferMap(input, output, drainer, app,
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
index 7c9da073..701e4e55 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
@@ -2,6 +2,8 @@
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
+import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 
@@ -13,9 +15,11 @@ public class DistributedDrainer extends AbstractDrainer {
 
 	StreamJitAppManager manager;
 
-	public DistributedDrainer(StreamJitAppManager manager) {
+	public DistributedDrainer(StreamJitApp app, TimeLogger logger,
+			StreamJitAppManager manager) {
+		super(app, logger);
 		this.manager = manager;
-		//TODO:
+		// TODO:
 		// Read this. Don't let the "this" reference escape during construction
 		// http://www.ibm.com/developerworks/java/library/j-jtp0618/
 		manager.setDrainer(this);

From bafd30a7c1a1d5f95ce9bb3e09d7aeac2a1bc5bf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 23:47:16 +0800
Subject: [PATCH 452/881] Sends DrainData after the final draining.

Considerable amount of draindata comes after the final draining. Controller
need to collect all and process them with a Interpreter blob.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 021d602a..6d30c462 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -432,7 +432,7 @@ private void drained() {
 			}
 			// System.out.println("Blob " + blobID + "is drained at mid");
 
-			if (drainType == DrainType.INTERMEDIATE) {
+			if (drainType != DrainType.DISCARD) {
 				SNMessageElement me;
 				if (crashed.get())
 					me = getEmptyDrainData();

From fe5ccba61f6c51af4577cd0c08a301ac72ea6e33 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 23:50:21 +0800
Subject: [PATCH 453/881] Running Interp blob after final draining disabled.

I disabled as it is not required for research experiments. Enable it when
running a real world application.
---
 src/edu/mit/streamjit/impl/common/AbstractDrainer.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 58c85be9..2bdb5f1d 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -269,7 +269,13 @@ public boolean drainFinal(Boolean isSemeFinal) {
 			e.printStackTrace();
 		}
 		logger.drainDataCollectionFinished("");
-		app.drainData = app.minimizeDrainData(app.drainData);
+		// TODO : Even after the final draining, we can clear some more
+		// intermediate data by running an Interpreter blob.
+		// StreamJitApp.minimizeDrainData() does this job. Uncomment the
+		// following lines later.
+		// app.drainData = app.minimizeDrainData(app.drainData);
+		// printDrainDataStats(app.drainData);
+		// dumpDrainData(app.drainData);
 		drainingDone(true);
 		stop();
 		return true;

From 129ffcc622efe06e8678ab76b19db8be3b08557f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 20 Dec 2014 23:59:19 +0800
Subject: [PATCH 454/881] Prints DrainData size only if size > 0.

---
 .../impl/distributed/node/BlobsManagerImpl.java     | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 6d30c462..129cc52f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -494,6 +494,7 @@ private SNDrainedData getSNDrainData() {
 					BoundaryChannel chanl = inputChannels.get(t);
 					ImmutableList<Object> draindata = chanl
 							.getUnprocessedData();
+					// if (draindata.size() > 0)
 					// System.out.println(String.format("From %s - %d",
 					// chanl.name(), draindata.size()));
 					inputDataBuilder.put(t, draindata);
@@ -511,6 +512,7 @@ private SNDrainedData getSNDrainData() {
 					BoundaryChannel chanl = outputChannels.get(t);
 					ImmutableList<Object> draindata = chanl
 							.getUnprocessedData();
+					// if (draindata.size() > 0)
 					// System.out.println(String.format("From %s - %d",
 					// chanl.name(), draindata.size()));
 					outputDataBuilder.put(t, draindata);
@@ -535,8 +537,9 @@ private void unprocessedDataFromLocalBuffer(
 			} else {
 				bufArray = bufferCleaner.copiedBuffer(t);
 			}
-			// System.out.println(String.format("From LocalBuffer: %s - %d", t,
-			// bufArray.length));
+			// if (bufArray.length > 0)
+			// System.out.println(String.format("From LocalBuffer: %s - %d",
+			// t, bufArray.length));
 			inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
 		}
 
@@ -576,8 +579,10 @@ private void printDrainDataStats(DrainData dd) {
 			System.out.println("**********printDrainDataStats*************");
 			if (dd != null) {
 				for (Token t : dd.getData().keySet()) {
-					System.out.println("From Blob: " + t.toString() + " - "
-							+ dd.getData().get(t).size());
+					int size = dd.getData().get(t).size();
+					if (size > 0)
+						System.out.println("From Blob: " + t.toString() + " - "
+								+ size);
 				}
 			}
 		}

From 9bf2f43896350acc0f6a234fd6d86bdec74a6e1c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 21 Dec 2014 08:42:10 +0800
Subject: [PATCH 455/881] Print msgs formatted.

---
 src/edu/mit/streamjit/impl/common/AbstractDrainer.java | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 2bdb5f1d..449f5f1e 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -217,7 +217,7 @@ public boolean drainIntermediate() {
 		logger.drainingStarted();
 		boolean state = startDraining(0);
 		if (!state) {
-			String msg = "Final drain has already been called. no more tuning.";
+			String msg = "Final drain has already been called. No more intermediate draining.";
 			System.err.println(msg);
 			logger.drainingFinished(msg);
 			return false;
@@ -248,9 +248,6 @@ public boolean drainFinal(Boolean isSemeFinal) {
 		logger.drainingStarted();
 		boolean state = startDraining(drainType);
 		if (!state) {
-			String msg = "Final drain has already been called.";
-			System.err.println(msg);
-			logger.drainingFinished(msg);
 			return false;
 		}
 

From 51bf5eb1ab7a3dc8db997bf637ffaad4663009fc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 21 Dec 2014 08:43:34 +0800
Subject: [PATCH 456/881] evaluateConfig() stops if no more input.

---
 .../impl/distributed/runtimer/OnlineTuner.java       | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 15632495..5bf577d0 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -283,10 +283,14 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 															// noise.
 				for (int i = 0; i < count; i++) {
 					ret = reconfigure(cfg);
-					writer.write(ret.second.toString());
-					writer.write('\n');
-					writer.flush();
-					total += ret.second;
+					if (ret.first) {
+						writer.write(ret.second.toString());
+						writer.write('\n');
+						writer.flush();
+						total += ret.second;
+					} else {
+						break;
+					}
 				}
 				double avg = total / count;
 				writer.write(String.format("Average execution time = %f%n\n",

From ff906570744570a745af960e066363184cc7620d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 21 Dec 2014 08:44:45 +0800
Subject: [PATCH 457/881] BugFix: Check StopWatch's state before stop it.

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index fc3e50ef..43e7f829 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -170,7 +170,7 @@ public void compilationStarted() {
 
 		@Override
 		public void drainingFinished(String msg) {
-			if (drainTimeSW != null) {
+			if (drainTimeSW != null && drainTimeSW.isRunning()) {
 				drainTimeSW.stop();
 				long time = drainTimeSW.elapsed(TimeUnit.MILLISECONDS);
 				logDrainTime(time);

From b69ab447e8485a555cabf498868ec2ff9b7615a9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 21 Dec 2014 08:56:02 +0800
Subject: [PATCH 458/881] Other agents call drainFinal().

HeadChannel, OnlineTuner and DistStreamCompiler call AbstractDrainer.drainFinal()
whenever they want to stop the app from running. (Input data exhausted or
Tuning finished)
---
 .../impl/distributed/DistributedStreamCompiler.java       | 3 ++-
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java   | 8 ++++----
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 3 ++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index eebb8223..285fbd92 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -268,7 +268,8 @@ private <I, O> boolean setBufferMap(Input<I> input, Output<O> output,
 									head) {
 								@Override
 								public void drain() {
-									drainer.startDraining(2);
+									// drainer.startDraining(2);
+									drainer.drainFinal(false);
 								}
 							});
 		} else {
diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index df1049ab..be2a404a 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -37,9 +37,8 @@ public static class AsyncHeadChannel extends AsyncOutputChannel {
 		private volatile boolean stopCalled;
 		private volatile boolean isFinal;
 
-		public AsyncHeadChannel(Buffer buffer,
-				ConnectionProvider conProvider, ConnectionInfo conInfo,
-				String bufferTokenName, int debugLevel) {
+		public AsyncHeadChannel(Buffer buffer, ConnectionProvider conProvider,
+				ConnectionInfo conInfo, String bufferTokenName, int debugLevel) {
 			super(conProvider, conInfo, bufferTokenName, debugLevel);
 			readBuffer = buffer;
 			stopCalled = false;
@@ -153,7 +152,8 @@ class DrainerThread extends Thread {
 
 			public void run() {
 				System.out.println("Input data finished");
-				drainer.startDraining(2);
+				// drainer.startDraining(2);
+				drainer.drainFinal(false);
 			}
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 5bf577d0..d69b9d3c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -253,7 +253,8 @@ private void handleTermination() throws IOException {
 
 	private void terminate() {
 		if (manager.isRunning()) {
-			drainer.startDraining(1);
+			// drainer.startDraining(1);
+			drainer.drainFinal(true);
 		} else {
 			manager.stop();
 		}

From f82353948a4bf3ef8e2ce40e3a721651fd5bdad6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 21 Dec 2014 09:48:51 +0800
Subject: [PATCH 459/881] Creates ExtraBuffer at Final draining.

If the compiler blob stores large amount of data during the init state,
then some other channels become full. So we need to create extra buffer even
at the final draining. Otherwise deadlock will cause.
---
 .../streamjit/impl/distributed/node/BlockingInputChannel.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 7853dfc7..5488d496 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -179,7 +179,7 @@ public void receiveData() {
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
-				if (stopType.get() > 1 && ++bufFullCount > 5) {
+				if (stopType.get() > 0 && ++bufFullCount > 20) {
 					this.extraBuffer = new ExtraBuffer();
 					extraBuffer.write(obj);
 					System.err
@@ -263,7 +263,7 @@ private void finalReceive() {
 						e.printStackTrace();
 					}
 
-					if (stopType.get() == 2 && ++bufFullCount > 5) {
+					if (++bufFullCount > 20) {
 						assert buffer != this.extraBuffer : "ExtraBuffer is full. This shouldn't be the case.";
 						assert this.extraBuffer == null : "Extra buffer has already been created.";
 						this.extraBuffer = new ExtraBuffer();

From 07319c2f68f5755d6e067113045b0d720fe4c20e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 09:05:59 +0800
Subject: [PATCH 460/881] cfg dir has been changed to appName/configurations

All app related output files and input files must be located at appName
directory.
---
 .../impl/common/ConfigurationAnalyzer.java    | 32 +++++++++----------
 .../impl/common/ConfigurationEditor.java      | 16 ++++++----
 .../DistributedStreamCompiler.java            |  8 +++--
 .../distributed/runtimer/OnlineTuner.java     | 13 +++++---
 .../streamjit/util/ConfigurationUtils.java    |  4 +--
 5 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index 8e695e3c..0e174b42 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -24,9 +24,9 @@ public static void main(String[] args) {
 		System.out.println(ca.getRunningTime("NestedSplitJoinCore", 3));
 	}
 
-	String appDir;
+	private final String cfgDir;
 
-	String appName;
+	private final String appName;
 
 	List<Integer> bestConfigurations;
 
@@ -45,14 +45,12 @@ public static void main(String[] args) {
 	 * </pre>
 	 */
 
-	private final String cfgDirectory = "configurations";
-
 	public ConfigurationAnalyzer(String appName) {
-		verifyPath(cfgDirectory, appName);
+		verifyPath("configurations", appName);
 		bestConfigurations = new LinkedList<>();
 		this.appName = appName;
-		this.appDir = String.format("%s%s%s", cfgDirectory, File.separator,
-				appName);
+		this.cfgDir = String.format("%s%s%s", appName, File.separator,
+				"configurations");
 	}
 
 	private void compare(FloatParameter p1, FloatParameter p2) {
@@ -70,8 +68,8 @@ private void compare(FloatParameter p1, FloatParameter p2) {
 	}
 
 	private void compare(Integer first, Integer second) {
-		Configuration cfg1 = readcoConfiguration(appDir, appName, first);
-		Configuration cfg2 = readcoConfiguration(appDir, appName, second);
+		Configuration cfg1 = readcoConfiguration(first);
+		Configuration cfg2 = readcoConfiguration(second);
 		for (Entry<String, Parameter> en : cfg1.getParametersMap().entrySet()) {
 			Parameter p1 = en.getValue();
 			Parameter p2 = cfg2.getParameter(en.getKey());
@@ -139,7 +137,9 @@ private SqliteAdapter connectDB(String appName) {
 	}
 
 	private double getRunningTime(String appName, int round) {
-		SqliteAdapter sqlite = connectDB(appName);
+		String dbPath = String.format("%s%s%s", appName, File.separator,
+				appName);
+		SqliteAdapter sqlite = connectDB(dbPath);
 		ResultSet result = sqlite.executeQuery(String.format(
 				"SELECT * FROM result WHERE id=%d", round));
 
@@ -152,22 +152,22 @@ private double getRunningTime(String appName, int round) {
 		return Double.parseDouble(runtime);
 	}
 
-	private Configuration readcoConfiguration(String appDir, String appName,
-			Integer cfgNo) {
-		String cfg = String.format("%s%s%d%s.cfg", appDir, File.separator,
+	private Configuration readcoConfiguration(Integer cfgNo) {
+		String cfg = String.format("%s%s%d%s.cfg", cfgDir, File.separator,
 				cfgNo, appName);
 		return ConfigurationUtils.readConfiguration(cfg);
 	}
 
 	private boolean verifyPath(String cfgDir, String appName) {
-		String dbPath = appName;
+		String dbPath = String.format("%s%s%s", appName, File.separator,
+				appName);
 		File db = new File(dbPath);
 		if (!db.exists())
 			throw new IllegalStateException("No database file found in "
 					+ dbPath);
 
-		String dirPath = String.format("%s%s%s", cfgDir, File.separator,
-				appName);
+		String dirPath = String.format("%s%s%s", appName, File.separator,
+				cfgDir);
 		File dir = new File(dirPath);
 		if (!dir.exists())
 			throw new IllegalStateException("No directory found in " + dirPath);
diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index e196e148..e90de2fb 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -4,7 +4,6 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
@@ -46,9 +45,11 @@ public static void main(String[] args) throws IOException {
 	 * Reads a configuration and changes its multiplier value.
 	 */
 	private static void changeMultiplierVal() {
-		String appName = "final_NestedSplitJoinCore";
+		String appName = "NestedSplitJoinCore";
+		String cfgFileName = "Final_NestedSplitJoinCore.cfg";
 		Configuration config = ConfigurationUtils.readConfiguration(String
-				.format("%s.cfg", appName));
+				.format("%s%sconfigurations%s%s", appName, File.separator,
+						File.separator, cfgFileName));
 		Configuration.Builder builder = Configuration.builder(config);
 		IntParameter mulParam = config.getParameter("multiplier",
 				IntParameter.class);
@@ -59,8 +60,7 @@ private static void changeMultiplierVal() {
 
 		IntParameter newMulParam = new IntParameter("multiplier", 1, 100, 100);
 		builder.addParameter(newMulParam);
-		ConfigurationUtils.saveConfg(builder.build(), "444",
-				"final_NestedSplitJoinCore");
+		ConfigurationUtils.saveConfg(builder.build(), "444", appName);
 
 	}
 
@@ -263,14 +263,16 @@ private static void print(String name) {
 	private static void convert() {
 		String appName = "ChannelVocoder7Kernel";
 		Configuration cfg = ConfigurationUtils.readConfiguration(String.format(
-				"%d%s.cfg", 1, appName));
+				"%s%sconfigurations%s%d%s.cfg", appName, File.separator,
+				File.separator, 1, appName));
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(
 					String.format("%d%s.cfg", 0, appName)));
 			String pythonDict = reader.readLine();
+			reader.close();
 
 			Configuration finalCfg = rebuildConfiguration(pythonDict, cfg);
-			ConfigurationUtils.saveConfg(finalCfg, "0", name);
+			ConfigurationUtils.saveConfg(finalCfg, "0", appName);
 		} catch (Exception ex) {
 			ex.printStackTrace();
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 285fbd92..fd5bcd98 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -154,10 +155,11 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		return cs;
 	}
 
-	private <I, O> Configuration cfgFromFile(OneToOneElement<I, O> stream,
+	private <I, O> Configuration cfgFromFile(StreamJitApp app,
 			Controller controller, Configuration defaultCfg) {
 		Configuration cfg1 = ConfigurationUtils.readConfiguration(String
-				.format("%s.cfg", stream.getClass().getSimpleName()));
+				.format("%s%sconfigurations%s%s.cfg", app.name, File.separator,
+						File.separator, app.name));
 		if (cfg1 == null) {
 			controller.closeAll();
 			throw new IllegalConfigurationException();
@@ -307,7 +309,7 @@ private <I, O> void setConfiguration(Controller controller,
 				throw new IllegalConfigurationException();
 			}
 		} else if (GlobalConstants.tune == 0) {
-			this.cfg = cfgFromFile(stream, controller, defaultCfg);
+			this.cfg = cfgFromFile(app, controller, defaultCfg);
 		} else
 			this.cfg = defaultCfg;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d69b9d3c..32f4a2e6 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -123,11 +123,13 @@ private void tune() {
 
 	private void evaluate() {
 		Configuration finalCfg = ConfigurationUtils.readConfiguration(String
-				.format("final_%s.cfg", app.name));
+				.format("%s%sconfigurations%sfinal_%s.cfg", app.name,
+						File.separator, File.separator, app.name));
 		evaluateConfig(finalCfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(String
-				.format("hand_%s.cfg", app.name));
+				.format("%s%sconfigurations%shand_%s.cfg", app.name,
+						File.separator, File.separator, app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		try {
@@ -151,7 +153,9 @@ private void verifyTuningTimes() {
 		for (int n : cfgNos) {
 			String cfgName = String.format("%d%s.cfg", n, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(String
-					.format("configurations/%s/%s", app.name, cfgName));
+					.format("%s%sconfigurations%s%s", app.name, File.separator,
+							File.separator, cfgName));
+
 			if (cfg == null)
 				continue;
 			evaluateConfig(cfg, cfgName);
@@ -236,7 +240,8 @@ private void handleTermination() throws IOException {
 		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(String
-				.format("hand_%s.cfg", app.name));
+				.format("%s%sconfigurations%shand_%s.cfg", app.name,
+						File.separator, File.separator, app.name));
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index f270fdf3..27545ec2 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -77,8 +77,8 @@ public static void saveConfg(String configJson, String namePrefix,
 			String appName) {
 		try {
 
-			File dir = new File(String.format("configurations%s%s",
-					File.separator, appName));
+			File dir = new File(String.format("%s%sconfigurations", appName,
+					File.separator));
 			if (!dir.exists())
 				if (!dir.mkdirs()) {
 					System.err.println("Make directory failed");

From bd38e07956b37d8c0fe845571cafa9eeb5bb5de4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:10:18 +0800
Subject: [PATCH 461/881] AbstractDrainer stores o/p files to app directory.

Name of the appDirectory is appName.
---
 .../mit/streamjit/impl/common/AbstractDrainer.java | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index 449f5f1e..ff50249a 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -2,6 +2,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -367,7 +368,9 @@ private void updateDrainDataStatistics(DrainData drainData) {
 
 	private void printDrainDataStats(DrainData drainData) {
 		try {
-			FileWriter writer = new FileWriter("draindatasize.txt", true);
+			String fileName = String.format("%s%sdraindatasize.txt", app.name,
+					File.separator);
+			FileWriter writer = new FileWriter(fileName, true);
 			writer.write("-----------------------------------------------------------\n");
 			System.out.println("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^");
 
@@ -385,10 +388,11 @@ private void printDrainDataStats(DrainData drainData) {
 			e.printStackTrace();
 		}
 	}
-
 	private void dumpDrainData(DrainData drainData) {
 		try {
-			FileOutputStream fout = new FileOutputStream("DrainData");
+			String fileName = String.format("%s%sDrainData", app.name,
+					File.separator);
+			FileOutputStream fout = new FileOutputStream(fileName);
 			ObjectOutputStream oos = new ObjectOutputStream(fout);
 			oos.writeObject(drainData);
 			oos.close();
@@ -409,7 +413,9 @@ public void dumpDraindataStatistics() throws IOException {
 			return;
 		}
 
-		FileWriter writer = new FileWriter("DrainDataStatistics.txt");
+		String fileName = String.format("%s%sDrainDataStatistics.txt",
+				app.name, File.separator);
+		FileWriter writer = new FileWriter(fileName);
 		for (Token t : drainDataStatistics.keySet()) {
 			writer.write(t.toString());
 			writer.write(" - ");

From e3f7324e02db864e485e340eeb3fd4f01fdfa310 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:13:16 +0800
Subject: [PATCH 462/881] FileTimeLogger stores o/p files to app directory.

Name of the appDirectory is appName.
---
 .../streamjit/impl/distributed/TimeLoggers.java   | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 43e7f829..78dc84b1 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -20,8 +21,8 @@
 public class TimeLoggers {
 
 	/**
-	 * Creates three files named prefix_compileTime.txt, prefix_runTime.txt and
-	 * prefix_drainTime.txt, and logs the time information.
+	 * Creates three files named compileTime.txt, runTime.txt and drainTime.txt
+	 * inside app.name directory, and logs the time information.
 	 * 
 	 * @author sumanan
 	 * @since Nov 25, 2014
@@ -42,10 +43,12 @@ private static FileWriter getFileWriter(String name) {
 		 * @param prefix
 		 *            : prefix for the file names.
 		 */
-		public FileTimeLogger(String prefix) {
-			super(getFileWriter(String.format("%s_compileTime.txt", prefix)),
-					getFileWriter(String.format("%s_runTime.txt", prefix)),
-					getFileWriter(String.format("%s_drainTime.txt", prefix)));
+		public FileTimeLogger(String appName) {
+			super(getFileWriter(String.format("%s%scompileTime.txt", appName,
+					File.separator)), getFileWriter(String.format(
+					"%s%srunTime.txt", appName, File.separator)),
+					getFileWriter(String.format("%s%sdrainTime.txt", appName,
+							File.separator)));
 		}
 	}
 

From a1702dff9e66a6b2f9a5b02be74d27e34b7a5352 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:15:27 +0800
Subject: [PATCH 463/881] MonitorBuffer stores o/p files to app directory.

Name of the appDirectory is appName.
---
 .../impl/distributed/node/BlobsManagerImpl.java       | 11 ++++++++---
 .../distributed/node/ConfigurationProcessorImpl.java  |  4 +++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 129cc52f..a1663a97 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.HashMap;
@@ -89,9 +90,11 @@ public class BlobsManagerImpl implements BlobsManager {
 	 */
 	private final boolean monitorBuffers = false;
 
+	private final String appName;
+
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
-			ConnectionProvider conProvider) {
+			ConnectionProvider conProvider, String appName) {
 		this.conInfoMap = conInfoMap;
 		this.streamNode = streamNode;
 		this.conProvider = conProvider;
@@ -100,6 +103,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
 		this.bufferManager = new SNLocalBufferManager(blobSet);
 
+		this.appName = appName;
 		bufferManager.initialise();
 		if (bufferManager.isbufferSizesReady())
 			createBEs(blobSet);
@@ -953,8 +957,9 @@ private class MonitorBuffers extends Thread {
 		public void run() {
 			FileWriter writter = null;
 			try {
-				writter = new FileWriter(String.format("BufferStatus%d.txt",
-						streamNode.getNodeID()), false);
+				String fileName = String.format("%s%sBufferStatus%d.txt",
+						appName, File.separator, streamNode.getNodeID());
+				writter = new FileWriter(fileName, false);
 
 				writter.write(String.format(
 						"********Started*************** - %d\n", id));
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 805d3faa..a7ca2705 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -98,8 +98,10 @@ private void processDynamicCfg(String json, DrainData drainData) {
 			Map<Token, ConnectionInfo> conInfoMap = (Map<Token, ConnectionInfo>) cfg
 					.getExtraData(GlobalConstants.CONINFOMAP);
 
+			String appName = (String) staticConfig
+					.getExtraData(GlobalConstants.TOPLEVEL_WORKER_NAME);
 			streamNode.setBlobsManager(new BlobsManagerImpl(blobSet,
-					conInfoMap, streamNode, conProvider));
+					conInfoMap, streamNode, conProvider, appName));
 		} else {
 			try {
 				streamNode.controllerConnection

From 38494a5430a61ef47f08bf9340a03134eb8d4616 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:16:50 +0800
Subject: [PATCH 464/881] ConfigurationEditor stores o/p files to app dir.

Name of the appDirectory is appName.
---
 .../mit/streamjit/impl/common/ConfigurationEditor.java | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index e90de2fb..c3a024b4 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -75,7 +75,10 @@ private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+		String appName = stream.getClass().getSimpleName();
+
+		name = String.format("%s%sconfigurations%shand_%s.cfg", appName,
+				File.separator, File.separator, appName);
 
 		try {
 			FileWriter writer = new FileWriter(name, false);
@@ -136,7 +139,10 @@ private static void generate1(OneToOneElement<?, ?> stream, int noOfnodes) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
+		String appName = stream.getClass().getSimpleName();
+
+		name = String.format("%s%sconfigurations%shand_%s.cfg", appName,
+				File.separator, File.separator, appName);
 
 		try {
 			FileWriter writer = new FileWriter(name, false);

From c59a3881a544f1f896895bc57a7ec7c43d832136 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:23:34 +0800
Subject: [PATCH 465/881] OnlineTuner writes EvalCfgs.txt to appDir.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 32f4a2e6..2872c69f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -279,8 +279,8 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		double total = 0;
 		int count = 4;
 		try {
-			writer = new FileWriter(String.format("Eval_%s.txt", app.name),
-					true);
+			writer = new FileWriter(String.format("%s%sEval_%s.txt", app.name,
+					File.separator, app.name), true);
 			writer.write("\n----------------------------------------\n");
 			writer.write(String.format("Configuration name = %s\n", cfgName));
 			if (cfg != null) {

From edc71a50c5d722494d873a1a7af4dcc04953daba Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:41:43 +0800
Subject: [PATCH 466/881] PerformanceLogger writes o/p to app dir.

---
 .../impl/distributed/StreamJitAppManager.java       |  2 +-
 .../mit/streamjit/impl/distributed/TailChannel.java | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 3e8c9e03..e137ef73 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -326,7 +326,7 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 		tailChannel = new TailChannel(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
 						+ tailToken.toString(), 0, skipCount,
-				GlobalConstants.outputCount);
+				GlobalConstants.outputCount, app.name);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 74b9b59b..75f05921 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
@@ -64,7 +65,7 @@ public class TailChannel extends BlockingInputChannel {
 	 */
 	public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
-			int skipCount, int steadyCount) {
+			int skipCount, int steadyCount, String appName) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 		this.skipCount = skipCount;
 		this.totalCount = steadyCount + skipCount;
@@ -74,7 +75,7 @@ public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 		skipLatch = new CountDownLatch(1);
 		this.skipLatchUp = true;
 		if (GlobalConstants.tune == 0) {
-			pLogger = new PerformanceLogger();
+			pLogger = new PerformanceLogger(appName);
 			pLogger.start();
 		}
 		printOutputCount();
@@ -178,15 +179,19 @@ private class PerformanceLogger extends Thread {
 
 		private AtomicBoolean stopFlag;
 
-		private PerformanceLogger() {
+		private final String appName;
+
+		private PerformanceLogger(String appName) {
 			stopFlag = new AtomicBoolean(false);
+			this.appName = appName;
 		}
 
 		public void run() {
 			int i = 0;
 			FileWriter writer;
 			try {
-				writer = new FileWriter("FixedOutPut.txt");
+				writer = new FileWriter(String.format("%s%sFixedOutPut.txt",
+						appName, File.separator));
 			} catch (IOException e1) {
 				e1.printStackTrace();
 				return;

From 9f345e1cfb6e0cdc6fc634d15f37d9a44a9f1822 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 10:58:41 +0800
Subject: [PATCH 467/881] TimeLogProcessor reads and writes from app dir

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 39 ++++++++++++-------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 819ea229..7a193f65 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -1,6 +1,7 @@
 package edu.mit.streamjit.util;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -16,11 +17,13 @@
 public class TimeLogProcessor {
 
 	public static void main(String[] args) throws IOException {
-		List<Integer> compileTime = processCompileTime();
-		List<Integer> runTime = processRunTime();
-		List<Integer> drainTime = processDrainTime();
+		String appName = "FMRadioCore";
+		List<Integer> compileTime = processCompileTime(appName);
+		List<Integer> runTime = processRunTime(appName);
+		List<Integer> drainTime = processDrainTime(appName);
 
-		FileWriter writer = new FileWriter("totalStats.txt");
+		FileWriter writer = new FileWriter(String.format("%s%stotalStats.txt",
+				appName, File.separator));
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 0; i < runTime.size(); i++) {
@@ -32,14 +35,16 @@ public static void main(String[] args) throws IOException {
 		}
 		writer.close();
 
-		writeHeapStat("st1.txt");
-		writeHeapStat("st2.txt");
+		writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
+		writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
 
-	private static List<Integer> processCompileTime() throws IOException {
+	private static List<Integer> processCompileTime(String appName)
+			throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
-				"FMRadioCore_compileTime.txt"));
-		FileWriter writer = new FileWriter("CompileTime.txt");
+				String.format("%s%scompileTime.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedCompileTime.txt", appName, File.separator));
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -59,11 +64,13 @@ private static List<Integer> processCompileTime() throws IOException {
 		writer.close();
 		return ret;
 	}
+	private static List<Integer> processRunTime(String appName)
+			throws IOException {
 
-	private static List<Integer> processRunTime() throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
-				"FMRadioCore_runTime.txt"));
-		FileWriter writer = new FileWriter("RunTime.txt");
+				String.format("%s%srunTime.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedRunTime.txt", appName, File.separator));
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -87,10 +94,12 @@ private static List<Integer> processRunTime() throws IOException {
 		return ret;
 	}
 
-	private static List<Integer> processDrainTime() throws IOException {
+	private static List<Integer> processDrainTime(String appName)
+			throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
-				"FMRadioCore_drainTime.txt"));
-		FileWriter writer = new FileWriter("DrainTime.txt");
+				String.format("%s%sDrainTime.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedDrainTime.txt", appName, File.separator));
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);

From 06384ec6e1bcec4fae09bac3f857b2517d20dacb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 12:55:19 +0800
Subject: [PATCH 468/881] DSCompiler creates appDir.

---
 .../impl/distributed/DistributedStreamCompiler.java |  2 ++
 .../streamjit/impl/distributed/common/Utils.java    | 13 +++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index fd5bcd98..f8ae91dd 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -40,6 +40,7 @@
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -133,6 +134,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		setConfiguration(controller, srcSink, stream, app, cfgManager,
 				conManager);
 
+		Utils.createAppDir(app.name);
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
 				cfgManager, conManager, logger);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index a64365ae..a7786313 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import java.io.File;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
@@ -61,4 +62,16 @@ public static void printOutOfMemory() {
 		System.out
 				.println("Memory Use :" + usedMemory + "M/" + maxMemory + "M");
 	}
+
+	public static void createAppDir(String appName) {
+		File appDir = new File(appName);
+		if (appDir.exists()) {
+			if (appDir.isDirectory())
+				return;
+			else
+				System.err.println("A file exists in the name of appDir-"
+						+ appName);
+		} else
+			appDir.mkdir();
+	}
 }

From 9134f110df36bbbc0c728f9a043e8e562fd6a7a0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 12:59:24 +0800
Subject: [PATCH 469/881] Working directory to OpenTuner process.

Working directory to the opentuner process can be set.
---
 .../distributed/runtimer/OnlineTuner.java     |  3 ++-
 src/edu/mit/streamjit/tuner/OpenTuner.java    |  4 +++-
 src/edu/mit/streamjit/tuner/TCPTuner.java     | 20 ++++++++++++-------
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2872c69f..0ebe62e3 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -62,7 +62,8 @@ private void tune() {
 		try {
 			tuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
-					File.separator, File.separator));
+					File.separator, File.separator),
+					new File(System.getProperty("user.dir")));
 
 			tuner.writeLine("program");
 			tuner.writeLine(app.name);
diff --git a/src/edu/mit/streamjit/tuner/OpenTuner.java b/src/edu/mit/streamjit/tuner/OpenTuner.java
index 10c794f3..2774511e 100644
--- a/src/edu/mit/streamjit/tuner/OpenTuner.java
+++ b/src/edu/mit/streamjit/tuner/OpenTuner.java
@@ -1,5 +1,6 @@
 package edu.mit.streamjit.tuner;
 
+import java.io.File;
 import java.io.IOException;
 
 /**
@@ -37,7 +38,8 @@ public interface OpenTuner {
 	 * 
 	 * @throws IOException
 	 */
-	public void startTuner(String tunerPath) throws IOException;
+	public void startTuner(String tunerPath, File workingDir)
+			throws IOException;
 
 	/**
 	 * Stop the Autotuner instance.
diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index 494aef50..bc508920 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -2,6 +2,7 @@
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -48,22 +49,26 @@ public boolean isAlive() {
 	}
 
 	@Override
-	public void startTuner(String tunerPath) throws IOException {
+	public void startTuner(String tunerPath, File workingDir)
+			throws IOException {
 		int min = 5000;
 		Random rand = new Random();
 		Integer port = rand.nextInt(65535 - min) + min;
 		if (GlobalConstants.tunerStartMode == 0) {
-			this.tuner = new ProcessBuilder("xterm", "-e", "python", tunerPath,
-					port.toString()).start();
+			ProcessBuilder pb = new ProcessBuilder("xterm", "-e", "python",
+					tunerPath, port.toString());
+			pb.directory(workingDir);
+			this.tuner = pb.start();
 		} else if (GlobalConstants.tunerStartMode == 1) {
-			this.tuner = new ProcessBuilder("python", tunerPath,
-					port.toString()).start();
+			ProcessBuilder pb = new ProcessBuilder("python", tunerPath,
+					port.toString());
+			pb.directory(workingDir);
+			this.tuner = pb.start();
 		} else
 			port = 12563;
 		this.connection = new TunerConnection();
 		connection.connect(port);
 	}
-
 	@Override
 	public void stopTuner() throws IOException {
 		if (tuner == null)
@@ -182,7 +187,8 @@ public static void main(String[] args) throws InterruptedException,
 
 		OpenTuner tuner = new TCPTuner();
 		try {
-			tuner.startTuner("/lib/opentuner/streamjit/streamjit.py");
+			tuner.startTuner("/lib/opentuner/streamjit/streamjit.py", new File(
+					System.getProperty("user.dir")));
 		} catch (IOException e) {
 			e.printStackTrace();
 		}

From 461ba3d443be636582376b52d09e2e186b74ad31 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 13:03:47 +0800
Subject: [PATCH 470/881] Refactored: OnlineTuner.startTuner() added.

---
 .../distributed/runtimer/OnlineTuner.java     | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 0ebe62e3..854c673f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -60,17 +60,7 @@ else if (GlobalConstants.tune == 2)
 	private void tune() {
 		int round = 0;
 		try {
-			tuner.startTuner(String.format(
-					"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
-					File.separator, File.separator),
-					new File(System.getProperty("user.dir")));
-
-			tuner.writeLine("program");
-			tuner.writeLine(app.name);
-
-			tuner.writeLine("confg");
-			tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
-
+			startTuner();
 			Pair<Boolean, Long> ret;
 
 			System.out.println("New tune run.............");
@@ -122,6 +112,19 @@ private void tune() {
 			terminate();
 	}
 
+	private void startTuner() throws IOException {
+		tuner.startTuner(String.format(
+				"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
+				File.separator, File.separator),
+				new File(System.getProperty("user.dir")));
+
+		tuner.writeLine("program");
+		tuner.writeLine(app.name);
+
+		tuner.writeLine("confg");
+		tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
+	}
+
 	private void evaluate() {
 		Configuration finalCfg = ConfigurationUtils.readConfiguration(String
 				.format("%s%sconfigurations%sfinal_%s.cfg", app.name,

From 1485e3788e0889bc1c3ee9ccda6c61084d0a7a62 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 13:36:14 +0800
Subject: [PATCH 471/881] CorrectPath has been set to OpenTuner process.

---
 .../impl/distributed/runtimer/OnlineTuner.java        | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 854c673f..8bf18a51 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -113,10 +113,15 @@ private void tune() {
 	}
 
 	private void startTuner() throws IOException {
-		tuner.startTuner(String.format(
+		String relativeTunerPath = String.format(
 				"lib%sopentuner%sstreamjit%sstreamjit2.py", File.separator,
-				File.separator, File.separator),
-				new File(System.getProperty("user.dir")));
+				File.separator, File.separator);
+
+		String absoluteTunerPath = String.format("%s%s%s",
+				System.getProperty("user.dir"), File.separator,
+				relativeTunerPath);
+
+		tuner.startTuner(absoluteTunerPath, new File(app.name));
 
 		tuner.writeLine("program");
 		tuner.writeLine(app.name);

From ae04d4d9f98a0aad17daca3f69f048aefe4ccb4c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 13:41:55 +0800
Subject: [PATCH 472/881] File name corrected.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 7a193f65..dc4745ad 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -97,7 +97,7 @@ private static List<Integer> processRunTime(String appName)
 	private static List<Integer> processDrainTime(String appName)
 			throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
-				String.format("%s%sDrainTime.txt", appName, File.separator)));
+				String.format("%s%sdrainTime.txt", appName, File.separator)));
 		FileWriter writer = new FileWriter(String.format(
 				"%s%sProcessedDrainTime.txt", appName, File.separator));
 		String line;

From 69269e92e6f6ebe808e449755abc4ba74ef589db Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 15:21:47 +0800
Subject: [PATCH 473/881] OfflineTuner passes working directory.

---
 src/edu/mit/streamjit/tuner/OfflineTuner.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/OfflineTuner.java b/src/edu/mit/streamjit/tuner/OfflineTuner.java
index f30fcb2d..b226e050 100644
--- a/src/edu/mit/streamjit/tuner/OfflineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OfflineTuner.java
@@ -90,7 +90,8 @@ public void tune(Benchmark app) throws InterruptedException {
 		try {
 			autoTuner.startTuner(String.format(
 					"lib%sopentuner%sstreamjit%sstreamjit.py", File.separator,
-					File.separator, File.separator));
+					File.separator, File.separator),
+					new File(System.getProperty("user.dir")));
 
 			autoTuner.writeLine("program");
 			autoTuner.writeLine(app.toString());

From 2ece33c63415e54b9a1728aff00100594ffdfe41 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 23:22:50 +0800
Subject: [PATCH 474/881] Starts OpenTuner python script in gnome-term.

---
 src/edu/mit/streamjit/tuner/TCPTuner.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index bc508920..3cb2fbc6 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -55,10 +55,12 @@ public void startTuner(String tunerPath, File workingDir)
 		Random rand = new Random();
 		Integer port = rand.nextInt(65535 - min) + min;
 		if (GlobalConstants.tunerStartMode == 0) {
-			ProcessBuilder pb = new ProcessBuilder("xterm", "-e", "python",
-					tunerPath, port.toString());
-			pb.directory(workingDir);
-			this.tuner = pb.start();
+			ProcessBuilder xtermPB = new ProcessBuilder("xterm", "-e",
+					"python", tunerPath, port.toString());
+			ProcessBuilder gnomePB = new ProcessBuilder("gnome-terminal", "-e",
+					String.format("python %s %s", tunerPath, port.toString()));
+			gnomePB.directory(workingDir);
+			this.tuner = gnomePB.start();
 		} else if (GlobalConstants.tunerStartMode == 1) {
 			ProcessBuilder pb = new ProcessBuilder("python", tunerPath,
 					port.toString());

From 112b16da05dd36ec18378d173f39a1c57b23c2d7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 22 Dec 2014 23:37:35 +0800
Subject: [PATCH 475/881] xterm holds after executing OpenTuner script.

---
 src/edu/mit/streamjit/tuner/TCPTuner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index 3cb2fbc6..8d508b87 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -55,7 +55,7 @@ public void startTuner(String tunerPath, File workingDir)
 		Random rand = new Random();
 		Integer port = rand.nextInt(65535 - min) + min;
 		if (GlobalConstants.tunerStartMode == 0) {
-			ProcessBuilder xtermPB = new ProcessBuilder("xterm", "-e",
+			ProcessBuilder xtermPB = new ProcessBuilder("xterm", "-hold", "-e",
 					"python", tunerPath, port.toString());
 			ProcessBuilder gnomePB = new ProcessBuilder("gnome-terminal", "-e",
 					String.format("python %s %s", tunerPath, port.toString()));

From cfb1c5b950f50a1e39311f377ca22d503116f4c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 23 Dec 2014 00:06:50 +0800
Subject: [PATCH 476/881] Ant:Targets run and buildAndRun have been added.

---
 jarapp.xml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index 7f65fe5a..463e4ab3 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -54,6 +54,24 @@
 		</scp>
 	</target>
 
+	<target name="buildAndRun" depends="buildApp, run">
+	</target>
+
+	<target name="run">
+		<exec dir="." executable="gnome-terminal" osfamily="unix">
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar ${app}.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+		</exec>
+	</target>
+
+
 	<target name="getLogin">
 		<input message="Enter Username:" addproperty="username" />
 		<input message="Enter Password:" addproperty="password">

From 38442565af1daad1a5ece7459dc881e3c07d97f6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 24 Dec 2014 00:22:07 +0800
Subject: [PATCH 477/881] FMRadio added to Ant's properties file.

---
 jarapp.properties | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jarapp.properties b/jarapp.properties
index e123880a..b0b25183 100644
--- a/jarapp.properties
+++ b/jarapp.properties
@@ -1,5 +1,7 @@
-app=NestedSplitJoin
-mainclass=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
+#app=NestedSplitJoin
+#mainclass=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
+app=FMRadio
+mainclass=edu.mit.streamjit.test.apps.fmradio.FMRadio
 username=sumanan
 ##password=abc123
 

From d9c742e8d2a6c9160f3bea7128a613b80c5ebac7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 24 Dec 2014 00:26:52 +0800
Subject: [PATCH 478/881] AsyncOutputChannel returns unprocessed data.

getUnprocessedData() returns empty ImmutableList.
---
 .../impl/distributed/node/AsyncOutputChannel.java         | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
index cf734983..c36f7369 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/AsyncOutputChannel.java
@@ -50,8 +50,7 @@ public void run() {
 				if (con == null || !con.isStillConnected()) {
 					try {
 						con = conProvider.getConnection(conInfo);
-						buffer = new AsyncTCPBuffer(
-								(AsyncTCPConnection) con);
+						buffer = new AsyncTCPBuffer((AsyncTCPConnection) con);
 					} catch (IOException e) {
 						e.printStackTrace();
 					}
@@ -62,13 +61,12 @@ public void run() {
 
 	@Override
 	public ImmutableList<Object> getUnprocessedData() {
-		return null;
+		return ImmutableList.of();
 	}
 
 	@Override
 	public void stop(boolean isFinal) {
-		while (con == null)
-			;
+		while (con == null);
 		this.isFinal = isFinal;
 		if (!stopCalled) {
 			try {

From eec439e6108414f3ab55ca4ca3a2c214942f3da0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 06:18:28 +0800
Subject: [PATCH 479/881] StreamJitApp creates AppDir.

This has been moved from DistStreamCompiler to StreamJitApp.
---
 .../streamjit/impl/distributed/DistributedStreamCompiler.java   | 2 --
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java        | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index f8ae91dd..fd5bcd98 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -40,7 +40,6 @@
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -134,7 +133,6 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		setConfiguration(controller, srcSink, stream, app, cfgManager,
 				conManager);
 
-		Utils.createAppDir(app.name);
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
 				cfgManager, conManager, logger);
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 0f600620..2dfac026 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -23,6 +23,7 @@
 import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.impl.interp.Interpreter;
@@ -90,6 +91,7 @@ public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
 		this.sink = sink;
 		this.jarFilePath = this.getClass().getProtectionDomain()
 				.getCodeSource().getLocation().getPath();
+		Utils.createAppDir(name);
 	}
 
 	/**

From 0d87c4df1d40e59c0f566a72a44c3fa82dff079c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 06:22:27 +0800
Subject: [PATCH 480/881] Visualizer and DotVisualizer have been added.

Visualizer interface can be used to visualize a stream graph and it's cfg.
DotVisualizer generates dot file and png file of the stream graph.
---
 .../impl/distributed/StreamJitApp.java        |   3 +
 .../impl/distributed/Visualizer.java          | 202 ++++++++++++++++++
 2 files changed, 205 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/Visualizer.java

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 2dfac026..2ab6b6b3 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -69,6 +69,8 @@ public class StreamJitApp {
 
 	public DrainData drainData = null;
 
+	public final Visualizer visualizer;
+
 	/**
 	 * Keeps track of assigned machine Ids of each blob. This information is
 	 * need for draining. TODO: If possible use a better solution.
@@ -92,6 +94,7 @@ public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
 		this.jarFilePath = this.getClass().getProtectionDomain()
 				.getCodeSource().getLocation().getPath();
 		Utils.createAppDir(name);
+		visualizer = new Visualizer.DotVisualizer(streamGraph);
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
new file mode 100644
index 00000000..861db936
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -0,0 +1,202 @@
+package edu.mit.streamjit.impl.distributed;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import edu.mit.streamjit.api.Filter;
+import edu.mit.streamjit.api.Joiner;
+import edu.mit.streamjit.api.OneToOneElement;
+import edu.mit.streamjit.api.Pipeline;
+import edu.mit.streamjit.api.Splitjoin;
+import edu.mit.streamjit.api.Splitter;
+import edu.mit.streamjit.api.StreamVisitor;
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Workers;
+
+/**
+ * Interface to visualize a stream graph and it's configurations. Use the
+ * constructor to get the stream graph.
+ * 
+ * @author Sumanan
+ * @since 29 Dec, 2014
+ */
+public interface Visualizer {
+
+	/**
+	 * Call this method with new configuration, whenever the configuration
+	 * changes.
+	 * 
+	 * @param cfg
+	 */
+	public void newConfiguration(Configuration cfg);
+
+	/**
+	 * Use this class to have no visualization.
+	 * 
+	 */
+	public static class NoVisualizer implements Visualizer {
+
+		@Override
+		public void newConfiguration(Configuration cfg) {
+			return;
+		}
+	}
+
+	/**
+	 * Generates dot file and then from the dot file generates graph.
+	 */
+	public static class DotVisualizer implements Visualizer {
+
+		protected final OneToOneElement<?, ?> streamGraph;
+
+		public DotVisualizer(OneToOneElement<?, ?> streamGraph) {
+			this.streamGraph = streamGraph;
+			String name = streamGraph.getClass().getSimpleName();
+			DOTstreamVisitor dotSV = new DOTstreamVisitor(name);
+			streamGraph.visit(dotSV);
+		}
+
+		/**
+		 * Visits through the Stream graph and generates dot file.
+		 * 
+		 * @author sumanan
+		 * @since 29 Dec, 2014
+		 */
+		private class DOTstreamVisitor extends StreamVisitor {
+
+			private final String streamJitAppname;
+			private final FileWriter writter;
+
+			DOTstreamVisitor(String streamJitAppname) {
+				this.streamJitAppname = streamJitAppname;
+				writter = fileWriter();
+			}
+
+			private FileWriter fileWriter() {
+				FileWriter w = null;
+				String fileName = String.format("%s%sgraph.dot",
+						streamJitAppname, File.separator);
+				try {
+					w = new FileWriter(fileName, false);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+				return w;
+			}
+
+			private void initilizeDot() {
+				try {
+					writter.write(String.format("digraph %s {\n",
+							streamJitAppname));
+					writter.write("\trankdir=TD;\n");
+					writter.write("\tnodesep=0.5;\n");
+					writter.write("\tranksep=equally;\n");
+					// writter.write("\tnode [shape = circle];\n");
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+
+			private void closeDot() {
+				try {
+					writter.write("}");
+					writter.flush();
+					writter.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+
+			private void runDot() {
+				String fileName = String.format("./%s%sgraph.dot",
+						streamJitAppname, File.separator);
+				String outFileName = String.format("./%s%sgraph.png",
+						streamJitAppname, File.separator);
+				ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng",
+						fileName, "-o", outFileName);
+				try {
+					Process p = pb.start();
+					p.waitFor();
+				} catch (IOException | InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+			@Override
+			public void beginVisit() {
+				initilizeDot();
+			}
+
+			@Override
+			public void visitFilter(Filter<?, ?> filter) {
+				updateDot(filter);
+			}
+
+			@Override
+			public boolean enterPipeline(Pipeline<?, ?> pipeline) {
+				return true;
+			}
+
+			@Override
+			public void exitPipeline(Pipeline<?, ?> pipeline) {
+
+			}
+
+			@Override
+			public boolean enterSplitjoin(Splitjoin<?, ?> splitjoin) {
+				return true;
+			}
+
+			@Override
+			public void visitSplitter(Splitter<?, ?> splitter) {
+				updateDot(splitter);
+			}
+
+			@Override
+			public boolean enterSplitjoinBranch(OneToOneElement<?, ?> element) {
+				return true;
+			}
+
+			@Override
+			public void exitSplitjoinBranch(OneToOneElement<?, ?> element) {
+			}
+
+			@Override
+			public void visitJoiner(Joiner<?, ?> joiner) {
+				updateDot(joiner);
+			}
+
+			@Override
+			public void exitSplitjoin(Splitjoin<?, ?> splitjoin) {
+			}
+
+			@Override
+			public void endVisit() {
+				closeDot();
+				runDot();
+			}
+
+			private void updateDot(Worker<?, ?> w) {
+				for (Worker<?, ?> suc : Workers.getSuccessors(w)) {
+					String first = w.getClass().getSimpleName();
+					String second = suc.getClass().getSimpleName();
+					int id = Workers.getIdentifier(w);
+					int sucID = Workers.getIdentifier(suc);
+					try {
+						writter.write(String.format("\t%d -> %d;\n", id, sucID));
+						// writter.write(String.format("\t%s -> %s;\n", first,
+						// second));
+					} catch (IOException e) {
+						e.printStackTrace();
+					}
+				}
+			}
+		}
+
+		@Override
+		public void newConfiguration(Configuration cfg) {
+
+		}
+	}
+}

From f0b3d22d924d2939cc161c81ea13f5ee0ecc2184 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:21:52 +0800
Subject: [PATCH 481/881] getter&setter added to StreamJitApp.configuration.

StreamJitApp.blobConfiguration has been made to private and getter, setter
methods have been introduced.
---
 .../impl/distributed/ConfigurationManager.java   |  2 +-
 .../impl/distributed/HotSpotTuning.java          |  2 +-
 .../streamjit/impl/distributed/StreamJitApp.java | 16 +++++++++++++++-
 .../impl/distributed/StreamJitAppManager.java    |  2 +-
 .../impl/distributed/WorkerMachine.java          |  2 +-
 .../impl/distributed/runtimer/OnlineTuner.java   |  2 +-
 6 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 96333cbb..4c1a1ae0 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -155,7 +155,7 @@ public Configuration getDynamicConfiguration() {
 			builder.addParameter(partParam.build());
 			if (GlobalConstants.useCompilerBlob)
 				builder.addSubconfiguration("blobConfigs",
-						app.blobConfiguration);
+						app.getConfiguration());
 			else
 				builder.addSubconfiguration("blobConfigs",
 						getInterpreterConfg());
diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index f77fbb8c..537d4d1b 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -60,7 +60,7 @@ public boolean newConfiguration(Configuration config) {
 		} catch (StreamCompilationFailedException ex) {
 			return false;
 		}
-		app.blobConfiguration = config;
+		app.setConfiguration(config);
 		return true;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 2ab6b6b3..9d13a9d2 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -82,7 +82,7 @@ public class StreamJitApp {
 	 * opentuner. Specifically, a {@link Configuration} that is generated by a
 	 * {@link BlobFactory#getDefaultConfiguration(java.util.Set)}.
 	 */
-	public Configuration blobConfiguration = null;
+	private Configuration blobConfiguration = null;
 
 	public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
 			Worker<?, ?> sink) {
@@ -259,4 +259,18 @@ private Set<Integer> getWorkerIds(List<Set<Worker<?, ?>>> blobList) {
 		}
 		return workerIds;
 	}
+
+	/**
+	 * @return the blobConfiguration
+	 */
+	public Configuration getConfiguration() {
+		return blobConfiguration;
+	}
+
+	/**
+	 * @param blobConfiguration the blobConfiguration to set
+	 */
+	public void setConfiguration(Configuration blobConfiguration) {
+		this.blobConfiguration = blobConfiguration;
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index e137ef73..642a5166 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -231,7 +231,7 @@ public boolean reconfigure(int multiplier) {
 		Configuration.Builder builder = Configuration.builder(cfgManager
 				.getDynamicConfiguration());
 
-		conInfoMap = conManager.conInfoMap(app.blobConfiguration,
+		conInfoMap = conManager.conInfoMap(app.getConfiguration(),
 				app.partitionsMachineMap, app.source, app.sink);
 
 		builder.putExtraData(GlobalConstants.CONINFOMAP, conInfoMap);
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 22b2d1e2..15c0144d 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -92,7 +92,7 @@ public boolean newConfiguration(Configuration config) {
 		} catch (StreamCompilationFailedException ex) {
 			return false;
 		}
-		app.blobConfiguration = config;
+		app.setConfiguration(config);
 		return true;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8bf18a51..16d4cd59 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -127,7 +127,7 @@ private void startTuner() throws IOException {
 		tuner.writeLine(app.name);
 
 		tuner.writeLine("confg");
-		tuner.writeLine(Jsonifiers.toJson(app.blobConfiguration).toString());
+		tuner.writeLine(Jsonifiers.toJson(app.getConfiguration()).toString());
 	}
 
 	private void evaluate() {

From 56dba6995862a58f6791c094e2f6bfbc87a2aa92 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:28:04 +0800
Subject: [PATCH 482/881] StreamJitApp.blobConfiguration -> configuration.

---
 .../impl/distributed/StreamJitApp.java        | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 9d13a9d2..124a678a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -78,11 +78,11 @@ public class StreamJitApp {
 	public Map<Token, Integer> blobtoMachineMap;
 
 	/**
-	 * blobConfiguration contains decision variables that are tuned by
-	 * opentuner. Specifically, a {@link Configuration} that is generated by a
-	 * {@link BlobFactory#getDefaultConfiguration(java.util.Set)}.
+	 * The latest valid {@link Configuration} that is received from OpenTuner.
+	 * {@link BlobFactory#getDefaultConfiguration(java.util.Set) generates the
+	 * initial configuration}.
 	 */
-	private Configuration blobConfiguration = null;
+	private Configuration configuration = null;
 
 	public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
 			Worker<?, ?> sink) {
@@ -261,16 +261,17 @@ private Set<Integer> getWorkerIds(List<Set<Worker<?, ?>>> blobList) {
 	}
 
 	/**
-	 * @return the blobConfiguration
+	 * @return the configuration
 	 */
 	public Configuration getConfiguration() {
-		return blobConfiguration;
+		return configuration;
 	}
 
 	/**
-	 * @param blobConfiguration the blobConfiguration to set
+	 * @param configuration
+	 *            the configuration to set
 	 */
-	public void setConfiguration(Configuration blobConfiguration) {
-		this.blobConfiguration = blobConfiguration;
+	public void setConfiguration(Configuration configuration) {
+		this.configuration = configuration;
 	}
 }

From e61aaef83070479bab0a9bf899d7c408db81851a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:31:06 +0800
Subject: [PATCH 483/881] Visualizer.newPartitionMachineMap() added.

Visualizer nees to know partition machine map to show blob configuration.
---
 .../impl/distributed/Visualizer.java          | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 861db936..143f36d9 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -3,6 +3,9 @@
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.Joiner;
@@ -32,6 +35,16 @@ public interface Visualizer {
 	 */
 	public void newConfiguration(Configuration cfg);
 
+	/**
+	 * Partitions Machine Map of the current configuration. Only the
+	 * {@link ConfigurationManager} has the information to generate this map.
+	 * Visualizer has no glue to generate this partitionsMachineMap.
+	 * 
+	 * @param partitionsMachineMap
+	 */
+	public void newPartitionMachineMap(
+			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap);
+
 	/**
 	 * Use this class to have no visualization.
 	 * 
@@ -42,6 +55,12 @@ public static class NoVisualizer implements Visualizer {
 		public void newConfiguration(Configuration cfg) {
 			return;
 		}
+
+		@Override
+		public void newPartitionMachineMap(
+				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+			return;
+		}
 	}
 
 	/**
@@ -198,5 +217,10 @@ private void updateDot(Worker<?, ?> w) {
 		public void newConfiguration(Configuration cfg) {
 
 		}
+
+		@Override
+		public void newPartitionMachineMap(
+				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+		}
 	}
 }

From ca41d2326d318327244766019179f8919bd53d7b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:33:35 +0800
Subject: [PATCH 484/881] StreamJitApp sets new variables to Visualizer.

Sets newConfiguration and newPartitionMachineMap.
---
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 124a678a..ef8849fd 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -273,5 +273,7 @@ public Configuration getConfiguration() {
 	 */
 	public void setConfiguration(Configuration configuration) {
 		this.configuration = configuration;
+		visualizer.newConfiguration(configuration);
+		visualizer.newPartitionMachineMap(partitionsMachineMap);
 	}
 }

From 42f423e69cf27ac4aca7f08c329d748c157252c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:45:34 +0800
Subject: [PATCH 485/881] Instance variable appName added to DotVisualizer.

---
 .../impl/distributed/Visualizer.java          | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 143f36d9..71fe3958 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -70,10 +70,12 @@ public static class DotVisualizer implements Visualizer {
 
 		protected final OneToOneElement<?, ?> streamGraph;
 
+		private final String appName;
+
 		public DotVisualizer(OneToOneElement<?, ?> streamGraph) {
 			this.streamGraph = streamGraph;
-			String name = streamGraph.getClass().getSimpleName();
-			DOTstreamVisitor dotSV = new DOTstreamVisitor(name);
+			this.appName = streamGraph.getClass().getSimpleName();
+			DOTstreamVisitor dotSV = new DOTstreamVisitor();
 			streamGraph.visit(dotSV);
 		}
 
@@ -85,18 +87,16 @@ public DotVisualizer(OneToOneElement<?, ?> streamGraph) {
 		 */
 		private class DOTstreamVisitor extends StreamVisitor {
 
-			private final String streamJitAppname;
 			private final FileWriter writter;
 
-			DOTstreamVisitor(String streamJitAppname) {
-				this.streamJitAppname = streamJitAppname;
+			DOTstreamVisitor() {
 				writter = fileWriter();
 			}
 
 			private FileWriter fileWriter() {
 				FileWriter w = null;
-				String fileName = String.format("%s%sgraph.dot",
-						streamJitAppname, File.separator);
+				String fileName = String.format("%s%sgraph.dot", appName,
+						File.separator);
 				try {
 					w = new FileWriter(fileName, false);
 				} catch (IOException e) {
@@ -107,8 +107,7 @@ private FileWriter fileWriter() {
 
 			private void initilizeDot() {
 				try {
-					writter.write(String.format("digraph %s {\n",
-							streamJitAppname));
+					writter.write(String.format("digraph %s {\n", appName));
 					writter.write("\trankdir=TD;\n");
 					writter.write("\tnodesep=0.5;\n");
 					writter.write("\tranksep=equally;\n");
@@ -129,10 +128,10 @@ private void closeDot() {
 			}
 
 			private void runDot() {
-				String fileName = String.format("./%s%sgraph.dot",
-						streamJitAppname, File.separator);
-				String outFileName = String.format("./%s%sgraph.png",
-						streamJitAppname, File.separator);
+				String fileName = String.format("./%s%sgraph.dot", appName,
+						File.separator);
+				String outFileName = String.format("./%s%sgraph.png", appName,
+						File.separator);
 				ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng",
 						fileName, "-o", outFileName);
 				try {

From f39b64ccba9d32b660db89228b626550578257bf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 07:48:01 +0800
Subject: [PATCH 486/881] Filename changed: graph.dot -> streamgraph.dot

---
 src/edu/mit/streamjit/impl/distributed/Visualizer.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 71fe3958..fc2a6bd6 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -95,7 +95,7 @@ private class DOTstreamVisitor extends StreamVisitor {
 
 			private FileWriter fileWriter() {
 				FileWriter w = null;
-				String fileName = String.format("%s%sgraph.dot", appName,
+				String fileName = String.format("%s%sstreamgraph.dot", appName,
 						File.separator);
 				try {
 					w = new FileWriter(fileName, false);
@@ -128,10 +128,10 @@ private void closeDot() {
 			}
 
 			private void runDot() {
-				String fileName = String.format("./%s%sgraph.dot", appName,
-						File.separator);
-				String outFileName = String.format("./%s%sgraph.png", appName,
-						File.separator);
+				String fileName = String.format("./%s%sstreamgraph.dot",
+						appName, File.separator);
+				String outFileName = String.format("./%s%sstreamgraph.png",
+						appName, File.separator);
 				ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng",
 						fileName, "-o", outFileName);
 				try {

From db65d5080d7a883877fd5706a0246af1cddff1d4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 15:18:27 +0800
Subject: [PATCH 487/881] DotVisualizer generates blobgraph.

---
 .../impl/distributed/Visualizer.java          | 98 ++++++++++++++++---
 1 file changed, 83 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index fc2a6bd6..2cfa2a49 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -1,8 +1,11 @@
 package edu.mit.streamjit.impl.distributed;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -15,8 +18,10 @@
 import edu.mit.streamjit.api.Splitter;
 import edu.mit.streamjit.api.StreamVisitor;
 import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 
 /**
  * Interface to visualize a stream graph and it's configurations. Use the
@@ -127,20 +132,6 @@ private void closeDot() {
 				}
 			}
 
-			private void runDot() {
-				String fileName = String.format("./%s%sstreamgraph.dot",
-						appName, File.separator);
-				String outFileName = String.format("./%s%sstreamgraph.png",
-						appName, File.separator);
-				ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng",
-						fileName, "-o", outFileName);
-				try {
-					Process p = pb.start();
-					p.waitFor();
-				} catch (IOException | InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
 			@Override
 			public void beginVisit() {
 				initilizeDot();
@@ -192,7 +183,7 @@ public void exitSplitjoin(Splitjoin<?, ?> splitjoin) {
 			@Override
 			public void endVisit() {
 				closeDot();
-				runDot();
+				runDot("streamgraph");
 			}
 
 			private void updateDot(Worker<?, ?> w) {
@@ -217,9 +208,86 @@ public void newConfiguration(Configuration cfg) {
 
 		}
 
+		private void runDot(String file) {
+			String fileName = String.format("./%s%s%s.dot", appName,
+					File.separator, file);
+			String outFileName = String.format("./%s%s%s.png", appName,
+					File.separator, file);
+			ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng", fileName,
+					"-o", outFileName);
+			try {
+				Process p = pb.start();
+				p.waitFor();
+			} catch (IOException | InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
 		@Override
 		public void newPartitionMachineMap(
 				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+			FileWriter writer;
+			try {
+				writer = blobGraphWriter();
+				for (int machine : partitionsMachineMap.keySet()) {
+					for (Set<Worker<?, ?>> blobworkers : partitionsMachineMap
+							.get(machine)) {
+						Token blobID = Utils.getblobID(blobworkers);
+						writer.write(String
+								.format("\tsubgraph \"cluster_%s\" { color="
+										+ "royalblue1; label = \"Blob-%s:Machine-%d\";",
+										blobID, blobID, machine));
+						Set<Integer> workerIDs = getWorkerIds(blobworkers);
+						for (Integer id : workerIDs)
+							writer.write(String.format(" %d;", id));
+						writer.write("}\n");
+					}
+				}
+				writer.write("}\n");
+				writer.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			runDot("blobgraph");
+		}
+
+		private Set<Integer> getWorkerIds(Set<Worker<?, ?>> blobworkers) {
+			Set<Integer> workerIds = new HashSet<>();
+			for (Worker<?, ?> w : blobworkers) {
+				workerIds.add(Workers.getIdentifier(w));
+			}
+			return workerIds;
+		}
+
+		/**
+		 * Copies all lines except the final closing bracket from
+		 * streamgraph.dot to blobgraph.dot.
+		 * 
+		 * @return
+		 * @throws IOException
+		 */
+		private FileWriter blobGraphWriter() throws IOException {
+			File streamGraph = new File(String.format("./%s%sstreamgraph.dot",
+					appName, File.separator));
+			File blobGraph = new File(String.format("./%s%sblobgraph.dot",
+					appName, File.separator));
+			BufferedReader reader = new BufferedReader(new FileReader(
+					streamGraph));
+			FileWriter writer = new FileWriter(blobGraph, false);
+			String line;
+			int unclosedParenthesis = 0;
+			while ((line = reader.readLine()) != null) {
+				if (line.contains("{"))
+					unclosedParenthesis++;
+				if (line.contains("}"))
+					unclosedParenthesis--;
+				if (unclosedParenthesis > 0) {
+					writer.write(line);
+					writer.write("\n");
+				}
+			}
+			reader.close();
+			return writer;
 		}
 	}
 }

From b096c1e285224b895b7e58132f380be060c7859e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 30 Dec 2014 15:56:10 +0800
Subject: [PATCH 488/881] hadDot tells if the system has dot tool.

Stops generating dot file if no dot tool is in the system.
---
 .../streamjit/impl/distributed/Visualizer.java | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 2cfa2a49..d0030741 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -69,7 +69,9 @@ public void newPartitionMachineMap(
 	}
 
 	/**
-	 * Generates dot file and then from the dot file generates graph.
+	 * Generates dot file and then from the dot file generates graph. Before
+	 * using this class, ensure that Graphviz is properly installed in the
+	 * system.
 	 */
 	public static class DotVisualizer implements Visualizer {
 
@@ -77,9 +79,15 @@ public static class DotVisualizer implements Visualizer {
 
 		private final String appName;
 
+		/**
+		 * Tells whether the dot tool is installed in the system or not.
+		 */
+		private boolean hasDot;
+
 		public DotVisualizer(OneToOneElement<?, ?> streamGraph) {
 			this.streamGraph = streamGraph;
 			this.appName = streamGraph.getClass().getSimpleName();
+			hasDot = true;
 			DOTstreamVisitor dotSV = new DOTstreamVisitor();
 			streamGraph.visit(dotSV);
 		}
@@ -219,13 +227,18 @@ private void runDot(String file) {
 				Process p = pb.start();
 				p.waitFor();
 			} catch (IOException | InterruptedException e) {
-				e.printStackTrace();
+				System.err
+						.println("DotVisualizer: dot(Graphviz) tool is not properly installed in the system");
+				hasDot = false;
+				// e.printStackTrace();
 			}
 		}
 
 		@Override
 		public void newPartitionMachineMap(
 				Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
+			if (!hasDot)
+				return;
 			FileWriter writer;
 			try {
 				writer = blobGraphWriter();
@@ -250,7 +263,6 @@ public void newPartitionMachineMap(
 			}
 			runDot("blobgraph");
 		}
-
 		private Set<Integer> getWorkerIds(Set<Worker<?, ?>> blobworkers) {
 			Set<Integer> workerIds = new HashSet<>();
 			for (Worker<?, ?> w : blobworkers) {

From 354ad15ee2da3418eee41b1a919acfa273af3f81 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 05:32:05 +0800
Subject: [PATCH 489/881] StreamJitApp deals with generics <I,O>

---
 .../mit/streamjit/impl/distributed/StreamJitApp.java | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index ef8849fd..1ff1657a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -42,22 +42,22 @@
  * @author Sumanan sumanan@mit.edu
  * @since Oct 8, 2013
  */
-public class StreamJitApp {
+public class StreamJitApp<I, O> {
 
 	/**
 	 * Since this is final, lets make public
 	 */
 	public final String topLevelClass;
 
-	public final Worker<?, ?> source;
+	public final Worker<I, ?> source;
 
-	public final Worker<?, ?> sink;
+	public final Worker<?, O> sink;
 
 	public final String jarFilePath;
 
 	public final String name;
 
-	final OneToOneElement<?, ?> streamGraph;
+	final OneToOneElement<I, O> streamGraph;
 
 	public BlobGraph blobGraph;
 
@@ -84,8 +84,8 @@ public class StreamJitApp {
 	 */
 	private Configuration configuration = null;
 
-	public StreamJitApp(OneToOneElement<?, ?> streamGraph, Worker<?, ?> source,
-			Worker<?, ?> sink) {
+	public StreamJitApp(OneToOneElement<I, O> streamGraph, Worker<I, ?> source,
+			Worker<?, O> sink) {
 		this.streamGraph = streamGraph;
 		this.name = streamGraph.getClass().getSimpleName();
 		this.topLevelClass = streamGraph.getClass().getName();

From c0862251037d52c8e973748015af817a99525538 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 05:45:58 +0800
Subject: [PATCH 490/881] Refactored:visit() moved from DSC to StreamJitApp

Now the StreamJitApp connects and verifies the stream graph. Moreover, it finds
the source and sink itself.
---
 .../DistributedStreamCompiler.java            | 43 +----------------
 .../impl/distributed/StreamJitApp.java        | 48 +++++++++++++++++--
 2 files changed, 46 insertions(+), 45 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index fd5bcd98..129590d7 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -13,7 +13,6 @@
 import com.google.common.collect.ImmutableMap;
 
 import edu.mit.streamjit.api.CompiledStream;
-import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.Input;
 import edu.mit.streamjit.api.Input.ManualInput;
 import edu.mit.streamjit.api.OneToOneElement;
@@ -21,7 +20,6 @@
 import edu.mit.streamjit.api.Pipeline;
 import edu.mit.streamjit.api.Portal;
 import edu.mit.streamjit.api.Splitjoin;
-import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob.Token;
@@ -29,13 +27,11 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.InputBufferFactory;
 import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.OutputBufferFactory;
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.TimeLogger;
-import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
@@ -123,10 +119,9 @@ public DistributedStreamCompiler(int noOfnodes, Configuration cfg) {
 
 	public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 			Input<I> input, Output<O> output) {
-		Pair<Worker<I, ?>, Worker<?, O>> srcSink = visit(stream);
+		StreamJitApp<I, O> app = new StreamJitApp<>(stream);
 		Controller controller = establishController();
-		StreamJitApp app = new StreamJitApp(stream, srcSink.first,
-				srcSink.second);
+
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
@@ -175,27 +170,6 @@ private <I, O> Configuration cfgFromFile(StreamJitApp app,
 		return cfg1;
 	}
 
-	/**
-	 * TODO: Need to check for other default subtypes of {@link OneToOneElement}
-	 * s. Now only checks for first generation children.
-	 * 
-	 * @param stream
-	 * @throws StreamCompilationFailedException
-	 *             if stream is default subtype of OneToOneElement
-	 */
-	private <I, O> void checkforDefaultOneToOneElement(
-			OneToOneElement<I, O> stream) {
-
-		if (stream.getClass() == Pipeline.class
-				|| stream.getClass() == Splitjoin.class
-				|| stream.getClass() == Filter.class) {
-			throw new StreamCompilationFailedException(
-					"Default subtypes of OneToOneElement are not accepted for"
-							+ " compilation by this compiler. OneToOneElement"
-							+ " that passed should be unique");
-		}
-	}
-
 	private Controller establishController() {
 		Map<CommunicationType, Integer> conTypeCount = new HashMap<>();
 
@@ -337,19 +311,6 @@ private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
 		return false;
 	}
 
-	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
-			OneToOneElement<I, O> stream) {
-		checkforDefaultOneToOneElement(stream);
-		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
-		stream.visit(primitiveConnector);
-		Worker<I, ?> source = (Worker<I, ?>) primitiveConnector.getSource();
-		Worker<?, O> sink = (Worker<?, O>) primitiveConnector.getSink();
-
-		VerifyStreamGraph verifier = new VerifyStreamGraph();
-		stream.visit(verifier);
-		return new Pair<Worker<I, ?>, Worker<?, O>>(source, sink);
-	}
-
 	private static class DistributedCompiledStream implements CompiledStream {
 
 		AbstractDrainer drainer;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 1ff1657a..ead8ccae 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -9,7 +9,10 @@
 
 import com.google.common.collect.ImmutableMap;
 
+import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.OneToOneElement;
+import edu.mit.streamjit.api.Pipeline;
+import edu.mit.streamjit.api.Splitjoin;
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.AbstractReadOnlyBuffer;
@@ -20,13 +23,16 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.MessageConstraint;
+import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.impl.interp.Interpreter;
+import edu.mit.streamjit.util.Pair;
 
 /**
  * This class contains all information about the current streamJit application
@@ -84,13 +90,13 @@ public class StreamJitApp<I, O> {
 	 */
 	private Configuration configuration = null;
 
-	public StreamJitApp(OneToOneElement<I, O> streamGraph, Worker<I, ?> source,
-			Worker<?, O> sink) {
+	public StreamJitApp(OneToOneElement<I, O> streamGraph) {
 		this.streamGraph = streamGraph;
+		Pair<Worker<I, ?>, Worker<?, O>> srcSink = visit(streamGraph);
 		this.name = streamGraph.getClass().getSimpleName();
 		this.topLevelClass = streamGraph.getClass().getName();
-		this.source = source;
-		this.sink = sink;
+		this.source = srcSink.first;
+		this.sink = srcSink.second;
 		this.jarFilePath = this.getClass().getProtectionDomain()
 				.getCodeSource().getLocation().getPath();
 		Utils.createAppDir(name);
@@ -276,4 +282,38 @@ public void setConfiguration(Configuration configuration) {
 		visualizer.newConfiguration(configuration);
 		visualizer.newPartitionMachineMap(partitionsMachineMap);
 	}
+
+	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
+			OneToOneElement<I, O> stream) {
+		checkforDefaultOneToOneElement(stream);
+		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
+		stream.visit(primitiveConnector);
+		Worker<I, ?> source = (Worker<I, ?>) primitiveConnector.getSource();
+		Worker<?, O> sink = (Worker<?, O>) primitiveConnector.getSink();
+
+		VerifyStreamGraph verifier = new VerifyStreamGraph();
+		stream.visit(verifier);
+		return new Pair<Worker<I, ?>, Worker<?, O>>(source, sink);
+	}
+
+	/**
+	 * TODO: Need to check for other default subtypes of {@link OneToOneElement}
+	 * s. Now only checks for first generation children.
+	 * 
+	 * @param stream
+	 * @throws StreamCompilationFailedException
+	 *             if stream is default subtype of OneToOneElement
+	 */
+	private <I, O> void checkforDefaultOneToOneElement(
+			OneToOneElement<I, O> stream) {
+
+		if (stream.getClass() == Pipeline.class
+				|| stream.getClass() == Splitjoin.class
+				|| stream.getClass() == Filter.class) {
+			throw new StreamCompilationFailedException(
+					"Default subtypes of OneToOneElement are not accepted for"
+							+ " compilation by this compiler. OneToOneElement"
+							+ " that passed should be unique");
+		}
+	}
 }

From 4d41f17808ab786e37f9450c74a72594cdc22d42 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 05:50:10 +0800
Subject: [PATCH 491/881] Unnecessary method params removed

from DSC.setConfiguration().
---
 .../streamjit/impl/distributed/DistributedStreamCompiler.java  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 129590d7..bda816b8 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -266,8 +266,7 @@ public void drain() {
 	}
 
 	private <I, O> void setConfiguration(Controller controller,
-			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
-			OneToOneElement<I, O> stream, StreamJitApp app,
+			Pair<Worker<I, ?>, Worker<?, O>> srcSink, StreamJitApp<I, O> app,
 			ConfigurationManager cfgManager, ConnectionManager conManager) {
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));

From fba1c6acde53aa488cfbfb0d3bc22f6f931d5a56 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 05:58:36 +0800
Subject: [PATCH 492/881] DSC gets source and sinks from StreamJitApp

srcSink has been removed and all methods in DistributedStreamCompiler
refers StreamJitApp to get source and sink of the app.
---
 .../DistributedStreamCompiler.java            | 28 ++++++++-----------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index bda816b8..d5c3e8f6 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -125,8 +125,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
-		setConfiguration(controller, srcSink, stream, app, cfgManager,
-				conManager);
+		setConfiguration(controller, app, cfgManager, conManager);
 
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
@@ -135,9 +134,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 				manager);
 		drainer.setBlobGraph(app.blobGraph);
 
-		boolean needTermination = setBufferMap(input, output, drainer, app,
-				srcSink);
-		setConstrains(srcSink, app);
+		boolean needTermination = setBufferMap(input, output, drainer, app);
+		setConstrains(app);
 
 		manager.reconfigure(1);
 		CompiledStream cs = new DistributedCompiledStream(drainer);
@@ -225,8 +223,7 @@ private <I, O> void manualPartition(
 	 * Sets head and tail buffers.
 	 */
 	private <I, O> boolean setBufferMap(Input<I> input, Output<O> output,
-			final AbstractDrainer drainer, StreamJitApp app,
-			Pair<Worker<I, ?>, Worker<?, O>> srcSink) {
+			final AbstractDrainer drainer, StreamJitApp<I, O> app) {
 		// TODO: derive a algorithm to find good buffer size and use here.
 		Buffer head = InputBufferFactory.unwrap(input).createReadableBuffer(
 				10000);
@@ -256,22 +253,20 @@ public void drain() {
 		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
 				.<Token, Buffer> builder();
 
-		bufferMapBuilder
-				.put(Token.createOverallInputToken(srcSink.first), head);
-		bufferMapBuilder.put(Token.createOverallOutputToken(srcSink.second),
-				tail);
+		bufferMapBuilder.put(Token.createOverallInputToken(app.source), head);
+		bufferMapBuilder.put(Token.createOverallOutputToken(app.sink), tail);
 
 		app.bufferMap = bufferMapBuilder.build();
 		return needTermination;
 	}
 
 	private <I, O> void setConfiguration(Controller controller,
-			Pair<Worker<I, ?>, Worker<?, O>> srcSink, StreamJitApp<I, O> app,
-			ConfigurationManager cfgManager, ConnectionManager conManager) {
+			StreamJitApp<I, O> app, ConfigurationManager cfgManager,
+			ConnectionManager conManager) {
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));
 		Configuration defaultCfg = bf.getDefaultConfiguration(Workers
-				.getAllWorkersInGraph(srcSink.first));
+				.getAllWorkersInGraph(app.source));
 
 		if (this.cfg != null) {
 			if (!verifyCfg(defaultCfg, this.cfg)) {
@@ -289,12 +284,11 @@ private <I, O> void setConfiguration(Controller controller,
 		cfgManager.newConfiguration(this.cfg);
 	}
 
-	private <I, O> void setConstrains(Pair<Worker<I, ?>, Worker<?, O>> srcSink,
-			StreamJitApp app) {
+	private <I, O> void setConstrains(StreamJitApp<I, O> app) {
 		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
 		// context.
 		List<MessageConstraint> constraints = MessageConstraint
-				.findConstraints(srcSink.first);
+				.findConstraints(app.source);
 		Set<Portal<?>> portals = new HashSet<>();
 		for (MessageConstraint mc : constraints)
 			portals.add(mc.getPortal());

From c5b7a23f9348ac95a704a79213c49d18fba3fff0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 06:06:41 +0800
Subject: [PATCH 493/881] Refactored:setConstrains() moved from DSC to SJApp

StreamJitApp sets the constrains.
---
 .../distributed/DistributedStreamCompiler.java | 18 ------------------
 .../impl/distributed/StreamJitApp.java         | 18 +++++++++++++++++-
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index d5c3e8f6..e618e777 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -3,7 +3,6 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -18,7 +17,6 @@
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Output;
 import edu.mit.streamjit.api.Pipeline;
-import edu.mit.streamjit.api.Portal;
 import edu.mit.streamjit.api.Splitjoin;
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.api.Worker;
@@ -28,9 +26,7 @@
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.InputBufferFactory;
-import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.OutputBufferFactory;
-import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
@@ -135,7 +131,6 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		drainer.setBlobGraph(app.blobGraph);
 
 		boolean needTermination = setBufferMap(input, output, drainer, app);
-		setConstrains(app);
 
 		manager.reconfigure(1);
 		CompiledStream cs = new DistributedCompiledStream(drainer);
@@ -284,19 +279,6 @@ private <I, O> void setConfiguration(Controller controller,
 		cfgManager.newConfiguration(this.cfg);
 	}
 
-	private <I, O> void setConstrains(StreamJitApp<I, O> app) {
-		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
-		// context.
-		List<MessageConstraint> constraints = MessageConstraint
-				.findConstraints(app.source);
-		Set<Portal<?>> portals = new HashSet<>();
-		for (MessageConstraint mc : constraints)
-			portals.add(mc.getPortal());
-		for (Portal<?> portal : portals)
-			Portals.setConstraints(portal, constraints);
-		app.constraints = constraints;
-	}
-
 	private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
 		if (defaultCfg.getParametersMap().keySet()
 				.equals(cfg.getParametersMap().keySet()))
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index ead8ccae..81294d1d 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -12,6 +12,7 @@
 import edu.mit.streamjit.api.Filter;
 import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Pipeline;
+import edu.mit.streamjit.api.Portal;
 import edu.mit.streamjit.api.Splitjoin;
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
@@ -25,6 +26,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.MessageConstraint;
+import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
@@ -71,7 +73,7 @@ public class StreamJitApp<I, O> {
 
 	public ImmutableMap<Token, Buffer> bufferMap;
 
-	public List<MessageConstraint> constraints;
+	public final List<MessageConstraint> constraints;
 
 	public DrainData drainData = null;
 
@@ -99,6 +101,7 @@ public StreamJitApp(OneToOneElement<I, O> streamGraph) {
 		this.sink = srcSink.second;
 		this.jarFilePath = this.getClass().getProtectionDomain()
 				.getCodeSource().getLocation().getPath();
+		this.constraints = getConstrains();
 		Utils.createAppDir(name);
 		visualizer = new Visualizer.DotVisualizer(streamGraph);
 	}
@@ -316,4 +319,17 @@ private <I, O> void checkforDefaultOneToOneElement(
 							+ " that passed should be unique");
 		}
 	}
+
+	private List<MessageConstraint> getConstrains() {
+		// TODO: Copied form DebugStreamCompiler. Need to be verified for this
+		// context.
+		List<MessageConstraint> constraints = MessageConstraint
+				.findConstraints(source);
+		Set<Portal<?>> portals = new HashSet<>();
+		for (MessageConstraint mc : constraints)
+			portals.add(mc.getPortal());
+		for (Portal<?> portal : portals)
+			Portals.setConstraints(portal, constraints);
+		return constraints;
+	}
 }

From dc95faa1da4d8886fac7836082e27a75e5e4e38c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 06:11:33 +0800
Subject: [PATCH 494/881] Generic related changes

---
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 81294d1d..74b6b443 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -286,8 +286,7 @@ public void setConfiguration(Configuration configuration) {
 		visualizer.newPartitionMachineMap(partitionsMachineMap);
 	}
 
-	private <I, O> Pair<Worker<I, ?>, Worker<?, O>> visit(
-			OneToOneElement<I, O> stream) {
+	private Pair<Worker<I, ?>, Worker<?, O>> visit(OneToOneElement<I, O> stream) {
 		checkforDefaultOneToOneElement(stream);
 		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
 		stream.visit(primitiveConnector);
@@ -307,9 +306,7 @@ public void setConfiguration(Configuration configuration) {
 	 * @throws StreamCompilationFailedException
 	 *             if stream is default subtype of OneToOneElement
 	 */
-	private <I, O> void checkforDefaultOneToOneElement(
-			OneToOneElement<I, O> stream) {
-
+	private void checkforDefaultOneToOneElement(OneToOneElement<I, O> stream) {
 		if (stream.getClass() == Pipeline.class
 				|| stream.getClass() == Splitjoin.class
 				|| stream.getClass() == Filter.class) {

From 4db77eb46e2a89ed282d442c7d8d6633df1f8cd0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 06:14:50 +0800
Subject: [PATCH 495/881] Unnecessary args removed from DSC.manualPartition.

---
 .../impl/distributed/DistributedStreamCompiler.java      | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index e618e777..2c3df82f 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -40,7 +40,6 @@
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
 import edu.mit.streamjit.util.ConfigurationUtils;
-import edu.mit.streamjit.util.Pair;
 
 /**
  * 
@@ -143,7 +142,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		return cs;
 	}
 
-	private <I, O> Configuration cfgFromFile(StreamJitApp app,
+	private <I, O> Configuration cfgFromFile(StreamJitApp<I, O> app,
 			Controller controller, Configuration defaultCfg) {
 		Configuration cfg1 = ConfigurationUtils.readConfiguration(String
 				.format("%s%sconfigurations%s%s.cfg", app.name, File.separator,
@@ -202,15 +201,13 @@ private Controller establishController() {
 		return partitionsMachineMap;
 	}
 
-	private <I, O> void manualPartition(
-			Pair<Worker<I, ?>, Worker<?, O>> srcSink,
-			OneToOneElement<I, O> stream, StreamJitApp app) {
+	private <I, O> void manualPartition(StreamJitApp<I, O> app) {
 		Integer[] machineIds = new Integer[this.noOfnodes - 1];
 		for (int i = 0; i < machineIds.length; i++) {
 			machineIds[i] = i + 1;
 		}
 		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-				machineIds, stream, srcSink.first, srcSink.second);
+				machineIds, app.streamGraph, app.source, app.sink);
 		app.newPartitionMap(partitionsMachineMap);
 	}
 

From 3ab162868f20eb67dcf1bf301bba7b8b0884f9c2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 11:47:33 +0800
Subject: [PATCH 496/881] Refers generic version of StreamJitApp.

StreamJitApp is now generic. So other classes which refers StreamJitApp refers
generic version instead of using the raw version of StreamJitApp.
---
 .../impl/distributed/ConfigurationManager.java   | 16 +++++++---------
 .../impl/distributed/StreamJitAppManager.java    |  4 ++--
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 4c1a1ae0..44ca905f 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -40,8 +40,8 @@
  * accordingly.
  * </ol>
  * 
- * One can implement this interface to try different search space designs as
- * he want.
+ * One can implement this interface to try different search space designs as he
+ * want.
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Jan 16, 2014
@@ -97,13 +97,12 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 * @author Sumanan sumanan@mit.edu
 	 * @since Jan 17, 2014
 	 */
-	public static abstract class AbstractConfigurationManager
-			implements
-				ConfigurationManager {
+	public static abstract class AbstractConfigurationManager implements
+			ConfigurationManager {
 
-		protected final StreamJitApp app;
+		protected final StreamJitApp<?, ?> app;
 
-		AbstractConfigurationManager(StreamJitApp app) {
+		AbstractConfigurationManager(StreamJitApp<?, ?> app) {
 			this.app = app;
 		}
 
@@ -136,8 +135,7 @@ public Configuration getDynamicConfiguration() {
 			partParam.addBlobFactory(comp2Factory);
 			app.blobtoMachineMap = new HashMap<>();
 
-			BlobFactory bf = GlobalConstants.useCompilerBlob
-					? comp2Factory
+			BlobFactory bf = GlobalConstants.useCompilerBlob ? comp2Factory
 					: intFactory;
 			for (Integer machineID : app.partitionsMachineMap.keySet()) {
 				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 642a5166..937804fc 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -47,7 +47,7 @@
 
 public class StreamJitAppManager {
 
-	private final StreamJitApp app;
+	private final StreamJitApp<?, ?> app;
 
 	private AppStatusProcessorImpl apStsPro = null;
 
@@ -102,7 +102,7 @@ public class StreamJitAppManager {
 
 	private final SNTimeInfoProcessor timeInfoProcessor;
 
-	public StreamJitAppManager(Controller controller, StreamJitApp app,
+	public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
 			ConfigurationManager cfgManager, ConnectionManager conManager,
 			TimeLogger logger) {
 		this.controller = controller;

From 253d1bcfe468e3cb94f57c115408f8a4ff8b9086 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 31 Dec 2014 11:56:31 +0800
Subject: [PATCH 497/881] Changes in ConfigurationEditor due to the changes

in other classes ( StreamJitApp and DistributedBlobFactory)
---
 .../impl/common/ConfigurationEditor.java        | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index c3a024b4..5f7760da 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -18,6 +18,8 @@
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.ConnectionManager;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
@@ -126,18 +128,15 @@ private static void edit(String name, int maxWor)
 	}
 
 	private static void generate1(OneToOneElement<?, ?> stream, int noOfnodes) {
-		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
-		stream.visit(primitiveConnector);
-		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
-		Worker<?, ?> sink = (Worker<?, ?>) primitiveConnector.getSink();
-		noofwrks = Workers.getIdentifier(sink) + 1;
-
-		StreamJitApp app = new StreamJitApp(stream, source, sink);
+		StreamJitApp<?, ?> app = new StreamJitApp<>(stream);
+		noofwrks = Workers.getIdentifier(app.sink) + 1;
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, noOfnodes);
+		ConnectionManager conManger = new BlockingTCPNoParams(0);
+		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManger,
+				noOfnodes);
 
 		Configuration cfg = bf.getDefaultConfiguration(Workers
-				.getAllWorkersInGraph(source));
+				.getAllWorkersInGraph(app.source));
 
 		String appName = stream.getClass().getSimpleName();
 

From 3f1e09fc9960b534dead05a9fd63d5067ca31f00 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 14:13:32 +0800
Subject: [PATCH 498/881] Unnecessary steadyLatch.countDown(); reduced.

---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 75f05921..4cb937d6 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -35,6 +35,8 @@ public class TailChannel extends BlockingInputChannel {
 
 	private boolean skipLatchUp;
 
+	private boolean steadyLatchUp;
+
 	/**
 	 * Periodically prints no of outputs generated. See
 	 * {@link #printOutputCount()}.
@@ -74,6 +76,7 @@ public TailChannel(Buffer buffer, ConnectionProvider conProvider,
 		steadyLatch = new CountDownLatch(1);
 		skipLatch = new CountDownLatch(1);
 		this.skipLatchUp = true;
+		this.steadyLatchUp = true;
 		if (GlobalConstants.tune == 0) {
 			pLogger = new PerformanceLogger(appName);
 			pLogger.start();
@@ -91,8 +94,10 @@ public void receiveData() {
 			skipLatchUp = false;
 		}
 
-		if (count > totalCount)
+		if (steadyLatchUp && count > totalCount) {
 			steadyLatch.countDown();
+			steadyLatchUp = false;
+		}
 	}
 
 	@Override
@@ -159,6 +164,7 @@ public void reset() {
 		count = 0;
 		lastCount = 0;
 		skipLatchUp = true;
+		steadyLatchUp = true;
 	}
 
 	/**

From 7691dcce3249c62b627dfa3390ffac45e793a4a7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 14:18:28 +0800
Subject: [PATCH 499/881] TailChannel.reset() order changed.

First resets the counts then resets the skipLatch and finally resets the
steadyLatch.
---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 4cb937d6..1f193b15 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -157,13 +157,13 @@ public void run() {
 	}
 
 	public void reset() {
-		steadyLatch.countDown();
-		steadyLatch = new CountDownLatch(1);
-		skipLatch.countDown();
-		skipLatch = new CountDownLatch(1);
 		count = 0;
 		lastCount = 0;
+		skipLatch.countDown();
+		skipLatch = new CountDownLatch(1);
 		skipLatchUp = true;
+		steadyLatch.countDown();
+		steadyLatch = new CountDownLatch(1);
 		steadyLatchUp = true;
 	}
 

From 58a1c1f2360012c0870b51a1ddea95ad088dc43d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 14:28:06 +0800
Subject: [PATCH 500/881] Sets thread name "PerformanceLogger".

---
 .../mit/streamjit/impl/distributed/TailChannel.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 1f193b15..ff8acd75 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -168,11 +168,10 @@ public void reset() {
 	}
 
 	/**
-	 * This method is needed apart from {@link #reset()} because if a thread
-	 * calls (From {@link AppStatusProcessor#processERROR()} reset() method
-	 * instead of this method, the threads which are waiting on latches at
-	 * {@link #getFixedOutputTime()} will not be released properly because the
-	 * thread thread which is waiting on skipLatch will wait on steady latch.
+	 * We need this method apart from {@link #reset()}, because the
+	 * {@link #reset()} method creates the latches immediately after
+	 * countDown(). This causes the threads which are waiting on the latches at
+	 * {@link #getFixedOutputTime()} will not be released properly.
 	 */
 	public void releaseAll() {
 		steadyLatch.countDown();
@@ -188,6 +187,7 @@ private class PerformanceLogger extends Thread {
 		private final String appName;
 
 		private PerformanceLogger(String appName) {
+			super("PerformanceLogger");
 			stopFlag = new AtomicBoolean(false);
 			this.appName = appName;
 		}

From 95377b72eceb61aa2ff1072aa44faa4c6eaf491b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 14:39:02 +0800
Subject: [PATCH 501/881] PerformanceLogger logs the average running time.

---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index ff8acd75..8cee0b97 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -205,9 +205,13 @@ public void run() {
 
 			writeInitialInfo(writer);
 
+			Long sum = 0l;
+
 			while (++i < 10 && !stopFlag.get()) {
 				try {
 					Long time = getFixedOutputTime();
+
+					sum += time;
 					System.out.println("Execution time is " + time
 							+ " milli seconds");
 
@@ -222,6 +226,9 @@ public void run() {
 				}
 			}
 			try {
+				writer.write("Average = " + sum / (i - 1));
+				writer.write('\n');
+				writer.flush();
 				writer.close();
 			} catch (IOException e) {
 				e.printStackTrace();

From 6ad05083f8ad6d73cd407be9a3f680c8b32459c3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 19:59:08 +0800
Subject: [PATCH 502/881] TailChannel.getFixedOutputTime() reset the latches

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java  | 1 -
 src/edu/mit/streamjit/impl/distributed/TailChannel.java      | 5 +----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 937804fc..130611ff 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -345,7 +345,6 @@ private void start() {
 		controller.sendToAll(Command.START);
 
 		if (tailChannel != null) {
-			tailChannel.reset();
 			tailThread = new Thread(tailChannel.getRunnable(),
 					tailChannel.name());
 			tailThread.start();
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 8cee0b97..29a245f7 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -12,7 +12,6 @@
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
@@ -121,6 +120,7 @@ public void stop(DrainType type) {
 	 * @throws InterruptedException
 	 */
 	public long getFixedOutputTime() throws InterruptedException {
+		reset();
 		skipLatch.await();
 		Stopwatch stopwatch = Stopwatch.createStarted();
 		steadyLatch.await();
@@ -218,9 +218,6 @@ public void run() {
 					writer.write(time.toString());
 					writer.write('\n');
 					writer.flush();
-
-					reset();
-
 				} catch (InterruptedException | IOException e) {
 					e.printStackTrace();
 				}

From 0d736e33dbb4ded0a7a0457f4393fb0bbe35355a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 5 Jan 2015 20:30:24 +0800
Subject: [PATCH 503/881] Removed evaluateConfig()'s trial evaluation.

I added a trial evaluation because the first evaluation showed noise.
The change in the TailChannel has fixed the noise. So the trial evaluation
is removed now.
---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 16d4cd59..fd955c41 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -293,9 +293,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			writer.write("\n----------------------------------------\n");
 			writer.write(String.format("Configuration name = %s\n", cfgName));
 			if (cfg != null) {
-				Pair<Boolean, Long> ret = reconfigure(cfg); // often the first
-															// run shows huge
-															// noise.
+				Pair<Boolean, Long> ret;
 				for (int i = 0; i < count; i++) {
 					ret = reconfigure(cfg);
 					if (ret.first) {

From 0ba4141522aa1f13e8ad96b7cf9d80f79db9cbd4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 08:03:55 +0800
Subject: [PATCH 504/881] @Deprecated added to obsolete methods.

The methods related to convert python dict to configuration are obsolete.
We can remove these methods later.
---
 src/edu/mit/streamjit/impl/common/ConfigurationEditor.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 5f7760da..64158dd5 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -265,6 +265,7 @@ private static void print(String name) {
 		}
 	}
 
+	@Deprecated
 	private static void convert() {
 		String appName = "ChannelVocoder7Kernel";
 		Configuration cfg = ConfigurationUtils.readConfiguration(String.format(
@@ -299,6 +300,7 @@ private static void convert() {
 	 *            Old configuration object.
 	 * @return New configuration object with updated values from the pythonDict.
 	 */
+	@Deprecated
 	private static Configuration rebuildConfiguration(String pythonDict,
 			Configuration config) {
 		// System.out.println(pythonDict);
@@ -343,6 +345,7 @@ private static Configuration rebuildConfiguration(String pythonDict,
 	 * @param cfg
 	 * @return
 	 */
+	@Deprecated
 	private static String getConfigurationString(Configuration cfg) {
 		String s = Jsonifiers.toJson(cfg).toString();
 		String s1 = s.replaceAll("__class__", "ttttt");

From 6442dcbeb4010481314d4ef9211068fd24638ccb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 09:03:20 +0800
Subject: [PATCH 505/881] Overloaded readConfiguration() added.

---
 .../streamjit/util/ConfigurationUtils.java    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 27545ec2..feec1b06 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -1,5 +1,7 @@
 package edu.mit.streamjit.util;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
@@ -18,6 +20,25 @@
  */
 public class ConfigurationUtils {
 
+	/**
+	 * Reads configuration from ./appName/configurations/namePrefixappName.cfg
+	 * and returns it.
+	 * 
+	 * @param appName
+	 *            name of the streamJit app.
+	 * 
+	 * @param namePrefix
+	 *            prefix to add to the cfg file name.
+	 */
+	public static Configuration readConfiguration(String appName,
+			String namePrefix) {
+		checkNotNull(appName);
+		namePrefix = namePrefix == null ? "" : namePrefix;
+		String cfgFilePath = String.format("%s%sconfigurations%s%s%s.cfg",
+				appName, File.separator, File.separator, namePrefix, appName);
+		return readConfiguration(cfgFilePath);
+	}
+
 	/**
 	 * @param cfgFilePath
 	 *            path of the configuration file that need to be read.

From cf91a21ad93d9eb44625357f9f9490292b769d85 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 09:06:33 +0800
Subject: [PATCH 506/881] Comments modified to reflect recent changes.

All app related files are saved into ./appName dir.
---
 src/edu/mit/streamjit/util/ConfigurationUtils.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index feec1b06..0a9a6bf1 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -64,8 +64,9 @@ public static Configuration readConfiguration(String cfgFilePath) {
 	}
 
 	/**
-	 * Saves the config into configurations/appName directory. output _.cfg file
-	 * will be named as namePrefixappName.cfg.
+	 * Saves the configuration into
+	 * ./appName/configurations/namePrefixappName.cfg. output _.cfg file will be
+	 * named as namePrefixappName.cfg.
 	 * 
 	 * @param config
 	 *            {@link Configuration} that need to be saved.
@@ -82,8 +83,9 @@ public static void saveConfg(Configuration config, String namePrefix,
 	}
 
 	/**
-	 * Saves the configJson into configurations/appName directory. output _.cfg
-	 * file will be named as namePrefixappName.cfg.
+	 * Saves the configuration into
+	 * ./appName/configurations/namePrefixappName.cfg. output _.cfg file will be
+	 * named as namePrefixappName.cfg.
 	 * 
 	 * @param configJson
 	 *            Json representation of the {@link Configuration} that need to

From e2fb99642099dbc2447e3e9a8259e8bf88d60025 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 09:49:15 +0800
Subject: [PATCH 507/881] Uses CfgUtils.readCfg() to read cfgs

Uses ConfigurationUtils.readConfiguration(appName,namePrefix) to read
configurations.
---
 .../impl/common/ConfigurationEditor.java      | 50 ++++++-------------
 1 file changed, 14 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 64158dd5..9949b73e 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -48,10 +48,11 @@ public static void main(String[] args) throws IOException {
 	 */
 	private static void changeMultiplierVal() {
 		String appName = "NestedSplitJoinCore";
-		String cfgFileName = "Final_NestedSplitJoinCore.cfg";
-		Configuration config = ConfigurationUtils.readConfiguration(String
-				.format("%s%sconfigurations%s%s", appName, File.separator,
-						File.separator, cfgFileName));
+		String namePrefix = "final_";
+		Configuration config = ConfigurationUtils.readConfiguration(appName,
+				namePrefix);
+		if (config == null)
+			return;
 		Configuration.Builder builder = Configuration.builder(config);
 		IntParameter mulParam = config.getParameter("multiplier",
 				IntParameter.class);
@@ -63,7 +64,6 @@ private static void changeMultiplierVal() {
 		IntParameter newMulParam = new IntParameter("multiplier", 1, 100, 100);
 		builder.addParameter(newMulParam);
 		ConfigurationUtils.saveConfg(builder.build(), "444", appName);
-
 	}
 
 	private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {
@@ -92,18 +92,11 @@ private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {
 		}
 	}
 
-	private static void edit(String name, int maxWor)
+	private static void edit(String cfgFilePath, int maxWor)
 			throws NumberFormatException, IOException {
-		Configuration cfg;
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			cfg = Configuration.fromJson(json);
-			reader.close();
-		} catch (Exception ex) {
-			System.err.println("File reader error");
+		Configuration cfg = ConfigurationUtils.readConfiguration(cfgFilePath);
+		if (cfg == null)
 			return;
-		}
 
 		Configuration.Builder builder = Configuration.builder(cfg);
 		BufferedReader keyinreader = new BufferedReader(new InputStreamReader(
@@ -174,18 +167,11 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 		ConfigurationUtils.saveConfg(cfg, "0", name);
 	}
 
-	private static void edit1(String name, int maxWor)
+	private static void edit1(String cfgFilePath, int maxWor)
 			throws NumberFormatException, IOException {
-		Configuration cfg;
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			cfg = Configuration.fromJson(json);
-			reader.close();
-		} catch (Exception ex) {
-			System.err.println("File reader error");
+		Configuration cfg = ConfigurationUtils.readConfiguration(cfgFilePath);
+		if (cfg == null)
 			return;
-		}
 
 		Configuration.Builder builder = Configuration.builder(cfg);
 		BufferedReader keyinreader = new BufferedReader(new InputStreamReader(
@@ -243,18 +229,10 @@ private static void edit1(String name, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
-	private static void print(String name) {
-		Configuration cfg;
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(name));
-			String json = reader.readLine();
-			cfg = Configuration.fromJson(json);
-			reader.close();
-		} catch (Exception ex) {
-			System.err.println("File reader error");
+	private static void print(String cfgFilePath) {
+		Configuration cfg = ConfigurationUtils.readConfiguration(cfgFilePath);
+		if (cfg == null)
 			return;
-		}
-
 		for (Map.Entry<String, Parameter> en : cfg.getParametersMap()
 				.entrySet()) {
 			if (en.getValue() instanceof SwitchParameter<?>) {

From 006697193e484aadd6a5f96e29032de24b3f6760 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 10:02:16 +0800
Subject: [PATCH 508/881] Uses CfgUtils.saveCfg().

Uses ConfigurationUtils.saveConfg(cfg, namePrefix, appName) to write the
generated configuration.
---
 .../impl/common/ConfigurationEditor.java      | 52 +++++++------------
 1 file changed, 19 insertions(+), 33 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 9949b73e..e912afab 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -23,7 +23,9 @@
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
+import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
 import edu.mit.streamjit.util.ConfigurationUtils;
+import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 public class ConfigurationEditor {
@@ -36,7 +38,8 @@ public class ConfigurationEditor {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		// generate1(new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
+		Pair<String, Integer> ret = generate(
+				new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
 		// edit1(name, noofwrks);
 		// print("4366NestedSplitJoinCore.cfg");
 		// convert();
@@ -66,30 +69,22 @@ private static void changeMultiplierVal() {
 		ConfigurationUtils.saveConfg(builder.build(), "444", appName);
 	}
 
-	private static void generate(OneToOneElement<?, ?> stream, int noOfnodes) {
+	private static Pair<String, Integer> generate(OneToOneElement<?, ?> stream,
+			int noOfnodes) {
 		ConnectWorkersVisitor primitiveConnector = new ConnectWorkersVisitor();
 		stream.visit(primitiveConnector);
 		Worker<?, ?> source = (Worker<?, ?>) primitiveConnector.getSource();
 		Worker<?, ?> sink = (Worker<?, ?>) primitiveConnector.getSink();
-		noofwrks = Workers.getIdentifier(sink) + 1;
+		int noofwrks = Workers.getIdentifier(sink) + 1;
 
 		BlobFactory bf = new DistributedBlobFactory(noOfnodes);
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
 		String appName = stream.getClass().getSimpleName();
-
-		name = String.format("%s%sconfigurations%shand_%s.cfg", appName,
-				File.separator, File.separator, appName);
-
-		try {
-			FileWriter writer = new FileWriter(name, false);
-			writer.write(cfg.toJson());
-			writer.flush();
-			writer.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
+		String namePrefix = "hand_";
+		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
+		return new Pair<String, Integer>(appName, noofwrks);
 	}
 
 	private static void edit(String cfgFilePath, int maxWor)
@@ -120,9 +115,10 @@ private static void edit(String cfgFilePath, int maxWor)
 		System.out.println("Successfully updated");
 	}
 
-	private static void generate1(OneToOneElement<?, ?> stream, int noOfnodes) {
+	private static Pair<String, Integer> generate1(
+			OneToOneElement<?, ?> stream, int noOfnodes) {
 		StreamJitApp<?, ?> app = new StreamJitApp<>(stream);
-		noofwrks = Workers.getIdentifier(app.sink) + 1;
+		int noofwrks = Workers.getIdentifier(app.sink) + 1;
 		ConfigurationManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManger = new BlockingTCPNoParams(0);
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManger,
@@ -131,19 +127,9 @@ private static void generate1(OneToOneElement<?, ?> stream, int noOfnodes) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(app.source));
 
-		String appName = stream.getClass().getSimpleName();
-
-		name = String.format("%s%sconfigurations%shand_%s.cfg", appName,
-				File.separator, File.separator, appName);
-
-		try {
-			FileWriter writer = new FileWriter(name, false);
-			writer.write(cfg.toJson());
-			writer.flush();
-			writer.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
+		String namePrefix = "hand_";
+		ConfigurationUtils.saveConfg(cfg, namePrefix, app.name);
+		return new Pair<String, Integer>(app.name, noofwrks);
 	}
 
 	/**
@@ -162,9 +148,9 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(source));
 
-		name = String.format("hand_%s.cfg", stream.getClass().getSimpleName());
-
-		ConfigurationUtils.saveConfg(cfg, "0", name);
+		String appName = stream.getClass().getSimpleName();
+		String namePrefix = "hand_";
+		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
 	}
 
 	private static void edit1(String cfgFilePath, int maxWor)

From 1e3c8b9ab8f49bdaec4800bed75241cdb7340d2f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 10:09:52 +0800
Subject: [PATCH 509/881] ConfigurationEditor.edit() takes namePrefix

---
 .../impl/common/ConfigurationEditor.java      | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index e912afab..72e8365e 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -30,9 +30,6 @@
 
 public class ConfigurationEditor {
 
-	static String name;
-	static int noofwrks;
-
 	/**
 	 * @param args
 	 * @throws IOException
@@ -87,9 +84,10 @@ private static Pair<String, Integer> generate(OneToOneElement<?, ?> stream,
 		return new Pair<String, Integer>(appName, noofwrks);
 	}
 
-	private static void edit(String cfgFilePath, int maxWor)
+	private static void edit(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
-		Configuration cfg = ConfigurationUtils.readConfiguration(cfgFilePath);
+		Configuration cfg = ConfigurationUtils.readConfiguration(appName,
+				namePrefix);
 		if (cfg == null)
 			return;
 
@@ -109,9 +107,7 @@ private static void edit(String cfgFilePath, int maxWor)
 		}
 
 		cfg = builder.build();
-		FileWriter writer = new FileWriter(name);
-		writer.write(cfg.toJson());
-		writer.close();
+		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
 		System.out.println("Successfully updated");
 	}
 
@@ -153,9 +149,10 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
 	}
 
-	private static void edit1(String cfgFilePath, int maxWor)
+	private static void edit1(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
-		Configuration cfg = ConfigurationUtils.readConfiguration(cfgFilePath);
+		Configuration cfg = ConfigurationUtils.readConfiguration(appName,
+				namePrefix);
 		if (cfg == null)
 			return;
 
@@ -209,9 +206,7 @@ private static void edit1(String cfgFilePath, int maxWor)
 		}
 
 		cfg = builder.build();
-		FileWriter writer = new FileWriter(name);
-		writer.write(cfg.toJson());
-		writer.close();
+		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
 		System.out.println("Successfully updated");
 	}
 

From 8ad20644d3fb268e8ac959c9dda1f9124c962fad Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 10:55:23 +0800
Subject: [PATCH 510/881] Uses CfgUtils.readCfg() to read cfgs.

---
 .../impl/common/ConfigurationEditor.java      |  1 -
 .../DistributedStreamCompiler.java            |  5 ++---
 .../distributed/runtimer/OnlineTuner.java     | 21 ++++++++-----------
 3 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 72e8365e..110d8431 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -3,7 +3,6 @@
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Map;
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 2c3df82f..8ae76c9d 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -144,9 +144,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 	private <I, O> Configuration cfgFromFile(StreamJitApp<I, O> app,
 			Controller controller, Configuration defaultCfg) {
-		Configuration cfg1 = ConfigurationUtils.readConfiguration(String
-				.format("%s%sconfigurations%s%s.cfg", app.name, File.separator,
-						File.separator, app.name));
+		Configuration cfg1 = ConfigurationUtils.readConfiguration(app.name,
+				null);
 		if (cfg1 == null) {
 			controller.closeAll();
 			throw new IllegalConfigurationException();
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index fd955c41..9e5210c6 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -51,6 +51,7 @@ public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
 		else if (GlobalConstants.tune == 2)
+			// verifyTuningTimes();
 			evaluate();
 		else
 			System.err
@@ -131,14 +132,12 @@ private void startTuner() throws IOException {
 	}
 
 	private void evaluate() {
-		Configuration finalCfg = ConfigurationUtils.readConfiguration(String
-				.format("%s%sconfigurations%sfinal_%s.cfg", app.name,
-						File.separator, File.separator, app.name));
+		Configuration finalCfg = ConfigurationUtils.readConfiguration(app.name,
+				"final_");
 		evaluateConfig(finalCfg, "Final configuration");
 
-		Configuration handCfg = ConfigurationUtils.readConfiguration(String
-				.format("%s%sconfigurations%shand_%s.cfg", app.name,
-						File.separator, File.separator, app.name));
+		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
+				"hand_");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		try {
@@ -161,9 +160,8 @@ private void verifyTuningTimes() {
 				550, 600, 650, 700, 750, 800, 850, 900, 950, 1000 };
 		for (int n : cfgNos) {
 			String cfgName = String.format("%d%s.cfg", n, app.name);
-			Configuration cfg = ConfigurationUtils.readConfiguration(String
-					.format("%s%sconfigurations%s%s", app.name, File.separator,
-							File.separator, cfgName));
+			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
+					new Integer(n).toString());
 
 			if (cfg == null)
 				continue;
@@ -248,9 +246,8 @@ private void handleTermination() throws IOException {
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		evaluateConfig(finalcfg, "Final configuration");
 
-		Configuration handCfg = ConfigurationUtils.readConfiguration(String
-				.format("%s%sconfigurations%shand_%s.cfg", app.name,
-						File.separator, File.separator, app.name));
+		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
+				"hand_");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {

From ddf1912d69d477cd30429664d60cd1365ee92ab8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 19:41:24 +0800
Subject: [PATCH 511/881] Comments added to edit() in ConfigurationEditor.

There are two edit methods in ConfigurationEditor. The usage of those methods
have been added as comments.
---
 .../mit/streamjit/impl/common/ConfigurationEditor.java   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 110d8431..73e08fd6 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -22,6 +22,7 @@
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
+import edu.mit.streamjit.impl.distributed.WorkerMachine;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
@@ -83,6 +84,10 @@ private static Pair<String, Integer> generate(OneToOneElement<?, ?> stream,
 		return new Pair<String, Integer>(appName, noofwrks);
 	}
 
+	/**
+	 * This edit is for the configurations which are generated using
+	 * {@link WorkerMachine} as {@link ConfigurationManager}.
+	 */
 	private static void edit(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
 		Configuration cfg = ConfigurationUtils.readConfiguration(appName,
@@ -148,6 +153,10 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 		ConfigurationUtils.saveConfg(cfg, namePrefix, appName);
 	}
 
+	/**
+	 * This edit is for the configurations which are generated using
+	 * {@link HotSpotTuning} as {@link ConfigurationManager}.
+	 */
 	private static void edit1(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
 		Configuration cfg = ConfigurationUtils.readConfiguration(appName,

From f5b280927821c3a3801d4b1d3234bbb3c9b9abf0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 21:20:01 +0800
Subject: [PATCH 512/881] ConfigurationPrognosticator added

An interface ConfigurationPrognosticator and a concrete class NoPrognostication
have been added. ConfigurationPrognosticator prognosticates the configurations
given by the OpenTuner and tell whether a configuration is more likely to give
a better search objective improvement or not.
---
 .../runtimer/ConfigurationPrognosticator.java | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
new file mode 100644
index 00000000..3e829830
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
@@ -0,0 +1,42 @@
+package edu.mit.streamjit.impl.distributed.runtimer;
+
+import edu.mit.streamjit.impl.common.Configuration;
+
+/**
+ * Prognosticates the {@link Configuration}s given by the OpenTuner and tell
+ * whether a {@link Configuration} is more likely to give a better search
+ * objective improvement or not. Depends on the prognosticated information,
+ * {@link OnlineTuner} may reconfigure the application or reject the
+ * configuration. Currently, the search objective is performance optimization.
+ * In future, some other resource optimization objectives may be added (e.g.,
+ * Energy minimization).
+ * 
+ * @author sumanan
+ * @since 6 Jan, 2015
+ */
+public interface ConfigurationPrognosticator {
+
+	/**
+	 * Prognosticate a {@link Configuration} and tell whether a
+	 * {@link Configuration} is more likely to give a better search objective
+	 * improvement or not.
+	 * 
+	 * @param config
+	 * @return {@code true} iff the config is more likely to give a better
+	 *         search objective improvement.
+	 */
+	public boolean prognosticate(Configuration config);
+
+	/**
+	 * No Prognostication. The method {@link #prognosticate(Configuration)}
+	 * always returns {@code true}
+	 */
+	public static final class NoPrognostication implements
+			ConfigurationPrognosticator {
+
+		@Override
+		public boolean prognosticate(Configuration config) {
+			return true;
+		}
+	}
+}

From c4eb6417fabd452fb078a1830d0c9be3e76c3c57 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 6 Jan 2015 21:27:07 +0800
Subject: [PATCH 513/881] OnlineTuner refers generic version of StreamJitApp

StreamJitApp is now generic. So other classes which refers StreamJitApp refers
generic version instead of using the raw version of StreamJitApp.
---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9e5210c6..8a02a383 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -29,13 +29,13 @@ public class OnlineTuner implements Runnable {
 	private final AbstractDrainer drainer;
 	private final StreamJitAppManager manager;
 	private final OpenTuner tuner;
-	private final StreamJitApp app;
+	private final StreamJitApp<?, ?> app;
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
 	private final TimeLogger logger;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
-			StreamJitApp app, ConfigurationManager cfgManager,
+			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
 			TimeLogger logger, boolean needTermination) {
 		this.drainer = drainer;
 		this.manager = manager;

From ed08b255433cd75971931a52b4289c33b2a0c001 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 7 Jan 2015 19:26:42 +0800
Subject: [PATCH 514/881] GraphPropertyPrognosticator added

Uses the partitioned graph's properties to decide potentially good and bad
configurations. Assumes the cluster environment is homogeneous.
---
 .../runtimer/GraphPropertyPrognosticator.java | 127 ++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
new file mode 100644
index 00000000..0fb24984
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -0,0 +1,127 @@
+package edu.mit.streamjit.impl.distributed.runtimer;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+
+/**
+ * Assumes the cluster environment is homogeneous.
+ * 
+ * @author sumanan
+ * @since 7 Jan, 2015
+ */
+public class GraphPropertyPrognosticator implements ConfigurationPrognosticator {
+
+	private final StreamJitApp<?, ?> app;
+
+	private final ConfigurationManager cfgManager;
+
+	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
+			ConfigurationManager cfgManager) {
+		this.app = app;
+		this.cfgManager = cfgManager;
+	}
+
+	@Override
+	public boolean prognosticate(Configuration config) {
+		if (!cfgManager.newConfiguration(config))
+			return false;
+		if (bigToSmallBlobRatio() > 10)
+			return false;
+		if (loadRatio() > 5)
+			return false;
+		return true;
+	}
+
+	/**
+	 * @return The ratio between the number of workers in the largest blob and
+	 *         the number of workers in the smallest blob.
+	 */
+	private float bigToSmallBlobRatio() {
+		int min = Integer.MAX_VALUE;
+		int max = Integer.MIN_VALUE;
+		int currentBlobSize;
+		for (List<Set<Worker<?, ?>>> blobList : app.partitionsMachineMap
+				.values()) {
+			for (Set<Worker<?, ?>> blobWorkers : blobList) {
+				currentBlobSize = blobWorkers.size();
+				min = Math.min(min, currentBlobSize);
+				max = Math.max(max, currentBlobSize);
+			}
+		}
+		float blobRatio = ((float) max) / min;
+		System.out.println("blobRatio - " + blobRatio);
+		return blobRatio;
+	}
+
+	/**
+	 * @return The ratio between the highest number of workers assigned to a
+	 *         machine and the lowest number of workers assigned to a machine.
+	 */
+	private float loadRatio() {
+		int min = Integer.MAX_VALUE;
+		int max = Integer.MIN_VALUE;
+		int workersInCurrentNode;
+		for (List<Set<Worker<?, ?>>> blobList : app.partitionsMachineMap
+				.values()) {
+			workersInCurrentNode = 0;
+			for (Set<Worker<?, ?>> blobWorkers : blobList) {
+				workersInCurrentNode += blobWorkers.size();
+			}
+			min = Math.min(min, workersInCurrentNode);
+			max = Math.max(max, workersInCurrentNode);
+		}
+		float loadRatio = ((float) max) / min;
+		System.out.println("loadRatio - " + loadRatio);
+		return loadRatio;
+	}
+
+	/**
+	 * @return The ratio between the total number of blobs to the total nodes.
+	 */
+	private float blobToNodeRatio() {
+		int nodes = 0;
+		int blobs = 0;
+		for (List<Set<Worker<?, ?>>> blobList : app.partitionsMachineMap
+				.values()) {
+			nodes++;
+			blobs += blobList.size();
+		}
+		float blobNodeRatio = ((float) blobs) / nodes;
+		return blobNodeRatio;
+	}
+
+	/**
+	 * @return The ratio between the total channels in the stream graph to the
+	 *         {@link BoundaryChannel} in the current configuration.
+	 */
+	private float totalToBoundaryChannelRatio() {
+		int totalChannels = 0;
+		int boundaryChannels = 0;
+		for (Integer machineID : app.partitionsMachineMap.keySet()) {
+			List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
+					.get(machineID);
+			Set<Worker<?, ?>> allWorkers = new HashSet<>();
+			for (Set<Worker<?, ?>> blobWorkers : blobList) {
+				allWorkers.addAll(blobWorkers);
+			}
+
+			for (Worker<?, ?> w : allWorkers) {
+				for (Worker<?, ?> succ : Workers.getSuccessors(w)) {
+					totalChannels++;
+					if (!allWorkers.contains(succ))
+						boundaryChannels++;
+				}
+			}
+		}
+		float boundaryChannelRatio = ((float) totalChannels) / boundaryChannels;
+		return boundaryChannelRatio;
+	}
+}
\ No newline at end of file

From 5e40f2406c330a55e513a5e3d0516128da7140ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 7 Jan 2015 19:28:52 +0800
Subject: [PATCH 515/881] OnlineTuner uses GraphPropertyPrognosticator.

To filter out bad configurations.
---
 .../distributed/runtimer/OnlineTuner.java     | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8a02a383..4bf1f35f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -33,6 +33,7 @@ public class OnlineTuner implements Runnable {
 	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
 	private final TimeLogger logger;
+	private final ConfigurationPrognosticator prognosticator;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
@@ -44,6 +45,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
 		this.logger = logger;
+		this.prognosticator = new GraphPropertyPrognosticator(app, cfgManager);
 	}
 
 	@Override
@@ -89,13 +91,17 @@ private void tune() {
 					ConfigurationUtils.saveConfg(cfgJson,
 							new Integer(round).toString(), app.name);
 
-				ret = reconfigure(config);
-				if (ret.first) {
-					tuner.writeLine(new Double(ret.second).toString());
-				} else {
-					tuner.writeLine("exit");
-					break;
-				}
+				boolean possibleBetter = prognosticator.prognosticate(config);
+				if (possibleBetter) {
+					ret = reconfigure(config);
+					if (ret.first) {
+						tuner.writeLine(new Double(ret.second).toString());
+					} else {
+						tuner.writeLine("exit");
+						break;
+					}
+				} else
+					tuner.writeLine(new Double(-1).toString());
 			}
 
 		} catch (IOException e) {

From bad93090c4c0bedf4b7fb44031128a4d95081d7e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 7 Jan 2015 19:35:11 +0800
Subject: [PATCH 516/881] Spelling correction : writter -> writer.

---
 .../impl/distributed/Visualizer.java          | 24 ++++++-------
 .../distributed/node/BlobsManagerImpl.java    | 36 +++++++++----------
 src/edu/mit/streamjit/tuner/TCPTuner.java     | 14 ++++----
 3 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index d0030741..979344c3 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -100,10 +100,10 @@ public DotVisualizer(OneToOneElement<?, ?> streamGraph) {
 		 */
 		private class DOTstreamVisitor extends StreamVisitor {
 
-			private final FileWriter writter;
+			private final FileWriter writer;
 
 			DOTstreamVisitor() {
-				writter = fileWriter();
+				writer = fileWriter();
 			}
 
 			private FileWriter fileWriter() {
@@ -120,11 +120,11 @@ private FileWriter fileWriter() {
 
 			private void initilizeDot() {
 				try {
-					writter.write(String.format("digraph %s {\n", appName));
-					writter.write("\trankdir=TD;\n");
-					writter.write("\tnodesep=0.5;\n");
-					writter.write("\tranksep=equally;\n");
-					// writter.write("\tnode [shape = circle];\n");
+					writer.write(String.format("digraph %s {\n", appName));
+					writer.write("\trankdir=TD;\n");
+					writer.write("\tnodesep=0.5;\n");
+					writer.write("\tranksep=equally;\n");
+					// writer.write("\tnode [shape = circle];\n");
 				} catch (IOException e) {
 					e.printStackTrace();
 				}
@@ -132,9 +132,9 @@ private void initilizeDot() {
 
 			private void closeDot() {
 				try {
-					writter.write("}");
-					writter.flush();
-					writter.close();
+					writer.write("}");
+					writer.flush();
+					writer.close();
 				} catch (IOException e) {
 					e.printStackTrace();
 				}
@@ -201,8 +201,8 @@ private void updateDot(Worker<?, ?> w) {
 					int id = Workers.getIdentifier(w);
 					int sucID = Workers.getIdentifier(suc);
 					try {
-						writter.write(String.format("\t%d -> %d;\n", id, sucID));
-						// writter.write(String.format("\t%s -> %s;\n", first,
+						writer.write(String.format("\t%d -> %d;\n", id, sucID));
+						// writer.write(String.format("\t%s -> %s;\n", first,
 						// second));
 					} catch (IOException e) {
 						e.printStackTrace();
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index a1663a97..feac8cc0 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -955,13 +955,13 @@ private class MonitorBuffers extends Thread {
 		}
 
 		public void run() {
-			FileWriter writter = null;
+			FileWriter writer = null;
 			try {
 				String fileName = String.format("%s%sBufferStatus%d.txt",
 						appName, File.separator, streamNode.getNodeID());
-				writter = new FileWriter(fileName, false);
+				writer = new FileWriter(fileName, false);
 
-				writter.write(String.format(
+				writer.write(String.format(
 						"********Started*************** - %d\n", id));
 				while (!stopFlag.get()) {
 					try {
@@ -974,34 +974,34 @@ public void run() {
 						break;
 
 					if (blobExecuters == null) {
-						writter.write("blobExecuters are null...\n");
+						writer.write("blobExecuters are null...\n");
 						continue;
 					}
 
-					writter.write("----------------------------------\n");
+					writer.write("----------------------------------\n");
 					for (BlobExecuter be : blobExecuters.values()) {
-						writter.write("Status of blob " + be.blobID.toString()
+						writer.write("Status of blob " + be.blobID.toString()
 								+ "\n");
 
 						if (be.bufferMap == null) {
-							writter.write("Buffer map is null...\n");
+							writer.write("Buffer map is null...\n");
 							continue;
 						}
 
 						if (stopFlag.get())
 							break;
 
-						writter.write("Input channel details\n");
-						write(be, writter, true);
+						writer.write("Input channel details\n");
+						write(be, writer, true);
 
-						writter.write("Output channel details\n");
-						write(be, writter, false);
+						writer.write("Output channel details\n");
+						write(be, writer, false);
 					}
-					writter.write("----------------------------------\n");
-					writter.flush();
+					writer.write("----------------------------------\n");
+					writer.flush();
 				}
 
-				writter.write(String.format(
+				writer.write(String.format(
 						"********Stopped*************** - %d\n", id));
 			} catch (IOException e1) {
 				e1.printStackTrace();
@@ -1009,14 +1009,14 @@ public void run() {
 			}
 
 			try {
-				if (writter != null)
-					writter.close();
+				if (writer != null)
+					writer.close();
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
 		}
 
-		private void write(BlobExecuter be, FileWriter writter, boolean isIn)
+		private void write(BlobExecuter be, FileWriter writer, boolean isIn)
 				throws IOException {
 			Set<Token> tokenSet = tokenSet(be, isIn);
 			for (Token t : tokenSet) {
@@ -1033,7 +1033,7 @@ private void write(BlobExecuter be, FileWriter writter, boolean isIn)
 
 				String status = availableResource >= min ? "Firable"
 						: "NOT firable";
-				writter.write(t.toString() + "\tMin - " + min
+				writer.write(t.toString() + "\tMin - " + min
 						+ ",\tAvailableResource - " + availableResource + "\t"
 						+ status + "\n");
 			}
diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index 8d508b87..008f3313 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -100,7 +100,7 @@ public void stopTuner() throws IOException {
 	private final class TunerConnection {
 
 		private BufferedReader reader = null;
-		private BufferedWriter writter = null;
+		private BufferedWriter writer = null;
 		private Socket socket = null;
 		private boolean isconnected = false;
 
@@ -115,7 +115,7 @@ void connect(int port) throws IOException {
 					InputStream is = socket.getInputStream();
 					OutputStream os = socket.getOutputStream();
 					this.reader = new BufferedReader(new InputStreamReader(is));
-					this.writter = new BufferedWriter(
+					this.writer = new BufferedWriter(
 							new OutputStreamWriter(os));
 					isconnected = true;
 					break;
@@ -139,10 +139,10 @@ void connect(int port) throws IOException {
 		public void writeLine(String msg) throws IOException {
 			if (isStillConnected()) {
 				try {
-					writter.write(msg);
+					writer.write(msg);
 					if (msg.toCharArray()[msg.length() - 1] != '\n')
-						writter.write('\n');
-					writter.flush();
+						writer.write('\n');
+					writer.flush();
 				} catch (IOException ix) {
 					isconnected = false;
 					throw ix;
@@ -170,8 +170,8 @@ public final void closeConnection() {
 			try {
 				if (reader != null)
 					this.reader.close();
-				if (writter != null)
-					this.writter.close();
+				if (writer != null)
+					this.writer.close();
 				if (socket != null)
 					this.socket.close();
 			} catch (IOException ex) {

From caf3101becd1ee0a306888c5e87d3c4ec6f4d245 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 13:13:07 +0800
Subject: [PATCH 517/881] ConfigurationPrognosticator.time() has been added

ConfigurationPrognosticator.time() is an auxiliary method that can be used to
update a configuration's running time. This method has been added for data
analysis purpose.
---
 .../runtimer/ConfigurationPrognosticator.java        | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
index 3e829830..39d0ef72 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
@@ -27,6 +27,14 @@ public interface ConfigurationPrognosticator {
 	 */
 	public boolean prognosticate(Configuration config);
 
+	/**
+	 * An auxiliary method that can be used to update a configuration's running
+	 * time. Has been added for data analysis purpose.
+	 * 
+	 * @param time
+	 */
+	public void time(double time);
+
 	/**
 	 * No Prognostication. The method {@link #prognosticate(Configuration)}
 	 * always returns {@code true}
@@ -38,5 +46,9 @@ public static final class NoPrognostication implements
 		public boolean prognosticate(Configuration config) {
 			return true;
 		}
+
+		@Override
+		public void time(double time) {
+		}
 	}
 }

From 3fa6ac1146626f1e02e81386ac390770165e95c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 13:16:55 +0800
Subject: [PATCH 518/881] GraphPropertyPrognosticator logs graph properties

to a file.
---
 .../runtimer/GraphPropertyPrognosticator.java | 70 +++++++++++++++++--
 1 file changed, 66 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 0fb24984..7aad65f5 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -1,5 +1,8 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -23,20 +26,37 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	private final ConfigurationManager cfgManager;
 
+	private final FileWriter writer;
+
+	private int count = 0;
+
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
 			ConfigurationManager cfgManager) {
 		this.app = app;
 		this.cfgManager = cfgManager;
+		this.writer = fileWriter();
+		writeHeader(writer);
 	}
 
 	@Override
 	public boolean prognosticate(Configuration config) {
+		count++;
 		if (!cfgManager.newConfiguration(config))
 			return false;
-		if (bigToSmallBlobRatio() > 10)
-			return false;
-		if (loadRatio() > 5)
-			return false;
+		float bigToSmallBlobRatio = bigToSmallBlobRatio();
+		float loadRatio = loadRatio();
+		float blobToNodeRatio = blobToNodeRatio();
+		float BoundaryChannelRatio = totalToBoundaryChannelRatio();
+		try {
+			writer.write(String.format("\n%4d\t\t", count));
+			writer.write(String.format("%.2f\t\t", bigToSmallBlobRatio));
+			writer.write(String.format("%.2f\t\t", loadRatio));
+			writer.write(String.format("%.2f\t\t", blobToNodeRatio));
+			writer.write(String.format("%.2f\t\t", BoundaryChannelRatio));
+			writer.flush();
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
 		return true;
 	}
 
@@ -124,4 +144,46 @@ private float totalToBoundaryChannelRatio() {
 		float boundaryChannelRatio = ((float) totalChannels) / boundaryChannels;
 		return boundaryChannelRatio;
 	}
+
+	private FileWriter fileWriter() {
+		FileWriter w = null;
+		String fileName = String.format("%s%sGraphProperty.txt", app.name,
+				File.separator);
+		try {
+			w = new FileWriter(fileName, false);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return w;
+	}
+
+	private static void writeHeader(FileWriter writer) {
+		try {
+			writer.write(String.format("%.7s", "cfgID"));
+			writer.write("\t\t");
+			writer.write(String.format("%.7s", "bigToSmallBlobRatio"));
+			writer.write("\t\t");
+			writer.write(String.format("%.7s", "loadRatio"));
+			writer.write("\t\t");
+			writer.write(String.format("%.7s", "blobToNodeRatio"));
+			writer.write("\t\t");
+			writer.write(String.format("%.7s", "BoundaryChannelRatio"));
+			writer.write("\t\t");
+			writer.write(String.format("%.7s", "time"));
+			writer.write("\t\t");
+			writer.flush();
+		} catch (IOException e) {
+
+		}
+	}
+
+	@Override
+	public void time(double time) {
+		try {
+			writer.write(String.format("%.0f\t\t", time));
+			writer.flush();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
 }
\ No newline at end of file

From 9cb4906ce01e7982431372855b8573c1f6ddfd84 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 13:19:40 +0800
Subject: [PATCH 519/881] Updates running time to ConfigPrognosticator.

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 4bf1f35f..9f7d2868 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -95,6 +95,7 @@ private void tune() {
 				if (possibleBetter) {
 					ret = reconfigure(config);
 					if (ret.first) {
+						prognosticator.time(ret.second);
 						tuner.writeLine(new Double(ret.second).toString());
 					} else {
 						tuner.writeLine("exit");

From bde6379af560b899ddd4acec83aa3af9418dda95 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 13:23:16 +0800
Subject: [PATCH 520/881] OnlineTuner adds configPrefix to the configuration

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9f7d2868..9378d878 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -86,6 +86,8 @@ private void tune() {
 						"---------------------%d-------------------------",
 						++round));
 				Configuration config = Configuration.fromJson(cfgJson);
+				config = addConfigPrefix(config,
+						new Integer(round).toString());
 
 				if (GlobalConstants.saveAllConfigurations)
 					ConfigurationUtils.saveConfg(cfgJson,
@@ -320,4 +322,10 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			e1.printStackTrace();
 		}
 	}
+
+	private Configuration addConfigPrefix(Configuration config, String prefix) {
+		Configuration.Builder builder = Configuration.builder(config);
+		builder.putExtraData("configPrefix", prefix);
+		return builder.build();
+	}
 }
\ No newline at end of file

From 613252b1893bad6bd7d117b5dd04472e3e7cf989 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 13:30:12 +0800
Subject: [PATCH 521/881] ndleTermination() adds ConfigPrefix.

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 9378d878..2e277429 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -86,8 +86,7 @@ private void tune() {
 						"---------------------%d-------------------------",
 						++round));
 				Configuration config = Configuration.fromJson(cfgJson);
-				config = addConfigPrefix(config,
-						new Integer(round).toString());
+				config = addConfigPrefix(config, new Integer(round).toString());
 
 				if (GlobalConstants.saveAllConfigurations)
 					ConfigurationUtils.saveConfg(cfgJson,
@@ -154,7 +153,6 @@ private void evaluate() {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
-
 		terminate();
 	}
 
@@ -251,12 +249,13 @@ private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
 		ConfigurationUtils.saveConfg(finalConfg, "final_", app.name);
-
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
+		finalcfg = addConfigPrefix(finalcfg, "final_");
 		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
 				"hand_");
+		handCfg = addConfigPrefix(handCfg, "hand_");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {

From 00767db4e01b736e3579d05e040511c76ef01ed0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 18:47:11 +0800
Subject: [PATCH 522/881] mePrefixappName.cfg -> namePrefix_appName.cfg

---
 .../impl/common/ConfigurationAnalyzer.java     |  2 +-
 .../impl/common/ConfigurationEditor.java       |  6 +++---
 .../impl/common/TuningStatistics.java          |  2 +-
 .../impl/distributed/runtimer/OnlineTuner.java | 14 +++++++-------
 .../mit/streamjit/util/ConfigurationUtils.java | 18 +++++++++---------
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index 0e174b42..b10f197f 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -153,7 +153,7 @@ private double getRunningTime(String appName, int round) {
 	}
 
 	private Configuration readcoConfiguration(Integer cfgNo) {
-		String cfg = String.format("%s%s%d%s.cfg", cfgDir, File.separator,
+		String cfg = String.format("%s%s%d_%s.cfg", cfgDir, File.separator,
 				cfgNo, appName);
 		return ConfigurationUtils.readConfiguration(cfg);
 	}
diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 73e08fd6..cdc7fafe 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -38,7 +38,7 @@ public static void main(String[] args) throws IOException {
 		Pair<String, Integer> ret = generate(
 				new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
 		// edit1(name, noofwrks);
-		// print("4366NestedSplitJoinCore.cfg");
+		// print("4366_NestedSplitJoinCore.cfg");
 		// convert();
 		// changeMultiplierVal();
 	}
@@ -236,11 +236,11 @@ private static void print(String cfgFilePath) {
 	private static void convert() {
 		String appName = "ChannelVocoder7Kernel";
 		Configuration cfg = ConfigurationUtils.readConfiguration(String.format(
-				"%s%sconfigurations%s%d%s.cfg", appName, File.separator,
+				"%s%sconfigurations%s%d_%s.cfg", appName, File.separator,
 				File.separator, 1, appName));
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(
-					String.format("%d%s.cfg", 0, appName)));
+					String.format("%d_%s.cfg", 0, appName)));
 			String pythonDict = reader.readLine();
 			reader.close();
 
diff --git a/src/edu/mit/streamjit/impl/common/TuningStatistics.java b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
index 0d831738..d3e7d98a 100644
--- a/src/edu/mit/streamjit/impl/common/TuningStatistics.java
+++ b/src/edu/mit/streamjit/impl/common/TuningStatistics.java
@@ -23,7 +23,7 @@ public class TuningStatistics {
 	 * @throws IOException
 	 */
 	public static void main(String[] args) throws IOException {
-		// printCfgValues("1NestedSplitJoinCore.cfg");
+		// printCfgValues("1_NestedSplitJoinCore.cfg");
 		printAll("../Tuner layer/tuning-oopsla2014");
 
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2e277429..11de2376 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -141,11 +141,11 @@ private void startTuner() throws IOException {
 
 	private void evaluate() {
 		Configuration finalCfg = ConfigurationUtils.readConfiguration(app.name,
-				"final_");
+				"final");
 		evaluateConfig(finalCfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
-				"hand_");
+				"hand");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		try {
@@ -166,7 +166,7 @@ private void verifyTuningTimes() {
 		int[] cfgNos = { 10, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500,
 				550, 600, 650, 700, 750, 800, 850, 900, 950, 1000 };
 		for (int n : cfgNos) {
-			String cfgName = String.format("%d%s.cfg", n, app.name);
+			String cfgName = String.format("%d_%s.cfg", n, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
 					new Integer(n).toString());
 
@@ -248,14 +248,14 @@ private int getMultiplier(Configuration config) {
 	private void handleTermination() throws IOException {
 		String finalConfg = tuner.readLine();
 		System.out.println("Tuning finished");
-		ConfigurationUtils.saveConfg(finalConfg, "final_", app.name);
+		ConfigurationUtils.saveConfg(finalConfg, "final", app.name);
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
-		finalcfg = addConfigPrefix(finalcfg, "final_");
+		finalcfg = addConfigPrefix(finalcfg, "final");
 		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
-				"hand_");
-		handCfg = addConfigPrefix(handCfg, "hand_");
+				"hand");
+		handCfg = addConfigPrefix(handCfg, "hand");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 0a9a6bf1..70fbafb6 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -21,7 +21,7 @@
 public class ConfigurationUtils {
 
 	/**
-	 * Reads configuration from ./appName/configurations/namePrefixappName.cfg
+	 * Reads configuration from ./appName/configurations/namePrefix_appName.cfg
 	 * and returns it.
 	 * 
 	 * @param appName
@@ -34,7 +34,7 @@ public static Configuration readConfiguration(String appName,
 			String namePrefix) {
 		checkNotNull(appName);
 		namePrefix = namePrefix == null ? "" : namePrefix;
-		String cfgFilePath = String.format("%s%sconfigurations%s%s%s.cfg",
+		String cfgFilePath = String.format("%s%sconfigurations%s%s_%s.cfg",
 				appName, File.separator, File.separator, namePrefix, appName);
 		return readConfiguration(cfgFilePath);
 	}
@@ -65,8 +65,8 @@ public static Configuration readConfiguration(String cfgFilePath) {
 
 	/**
 	 * Saves the configuration into
-	 * ./appName/configurations/namePrefixappName.cfg. output _.cfg file will be
-	 * named as namePrefixappName.cfg.
+	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will be
+	 * named as namePrefix_appName.cfg.
 	 * 
 	 * @param config
 	 *            {@link Configuration} that need to be saved.
@@ -74,7 +74,7 @@ public static Configuration readConfiguration(String cfgFilePath) {
 	 *            prefix to add to the out put file name.
 	 * @param appName
 	 *            name of the streamJit app. output _.cfg file will be named as
-	 *            namePrefixappName.cfg.
+	 *            namePrefix_appName.cfg.
 	 */
 	public static void saveConfg(Configuration config, String namePrefix,
 			String appName) {
@@ -84,8 +84,8 @@ public static void saveConfg(Configuration config, String namePrefix,
 
 	/**
 	 * Saves the configuration into
-	 * ./appName/configurations/namePrefixappName.cfg. output _.cfg file will be
-	 * named as namePrefixappName.cfg.
+	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will be
+	 * named as namePrefix_appName.cfg.
 	 * 
 	 * @param configJson
 	 *            Json representation of the {@link Configuration} that need to
@@ -94,7 +94,7 @@ public static void saveConfg(Configuration config, String namePrefix,
 	 *            prefix to add to the out put file name.
 	 * @param appName
 	 *            name of the streamJit app. output _.cfg file will be named as
-	 *            namePrefixappName.cfg.
+	 *            namePrefix_appName.cfg.
 	 */
 	public static void saveConfg(String configJson, String namePrefix,
 			String appName) {
@@ -108,7 +108,7 @@ public static void saveConfg(String configJson, String namePrefix,
 					return;
 				}
 
-			File file = new File(dir, String.format("%s%s.cfg", namePrefix,
+			File file = new File(dir, String.format("%s_%s.cfg", namePrefix,
 					appName));
 			FileWriter writer = new FileWriter(file, false);
 			writer.write(configJson);

From bf9eb99ad7348b1f7aa239f0629becd5e5524044 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 19:09:49 +0800
Subject: [PATCH 523/881] addConfigPrefix() moved to ConfigurationUtils.java

addConfigPrefix() has been moved to ConfigurationUtils.java from
OnlineTuner.java.
---
 .../distributed/runtimer/OnlineTuner.java     | 13 +++-------
 .../streamjit/util/ConfigurationUtils.java    | 26 ++++++++++++++++---
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 11de2376..ac8e448c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -86,7 +86,8 @@ private void tune() {
 						"---------------------%d-------------------------",
 						++round));
 				Configuration config = Configuration.fromJson(cfgJson);
-				config = addConfigPrefix(config, new Integer(round).toString());
+				config = ConfigurationUtils.addConfigPrefix(config,
+						new Integer(round).toString());
 
 				if (GlobalConstants.saveAllConfigurations)
 					ConfigurationUtils.saveConfg(cfgJson,
@@ -250,12 +251,12 @@ private void handleTermination() throws IOException {
 		System.out.println("Tuning finished");
 		ConfigurationUtils.saveConfg(finalConfg, "final", app.name);
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
-		finalcfg = addConfigPrefix(finalcfg, "final");
+		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
 		evaluateConfig(finalcfg, "Final configuration");
 
 		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
 				"hand");
-		handCfg = addConfigPrefix(handCfg, "hand");
+		handCfg = ConfigurationUtils.addConfigPrefix(handCfg, "hand");
 		evaluateConfig(handCfg, "Handtuned configuration");
 
 		if (needTermination) {
@@ -321,10 +322,4 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			e1.printStackTrace();
 		}
 	}
-
-	private Configuration addConfigPrefix(Configuration config, String prefix) {
-		Configuration.Builder builder = Configuration.builder(config);
-		builder.putExtraData("configPrefix", prefix);
-		return builder.build();
-	}
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 70fbafb6..4cb7bfaf 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -65,8 +65,8 @@ public static Configuration readConfiguration(String cfgFilePath) {
 
 	/**
 	 * Saves the configuration into
-	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will be
-	 * named as namePrefix_appName.cfg.
+	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will
+	 * be named as namePrefix_appName.cfg.
 	 * 
 	 * @param config
 	 *            {@link Configuration} that need to be saved.
@@ -84,8 +84,8 @@ public static void saveConfg(Configuration config, String namePrefix,
 
 	/**
 	 * Saves the configuration into
-	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will be
-	 * named as namePrefix_appName.cfg.
+	 * ./appName/configurations/namePrefix_appName.cfg. output _.cfg file will
+	 * be named as namePrefix_appName.cfg.
 	 * 
 	 * @param configJson
 	 *            Json representation of the {@link Configuration} that need to
@@ -118,4 +118,22 @@ public static void saveConfg(String configJson, String namePrefix,
 			e.printStackTrace();
 		}
 	}
+
+	/**
+	 * Adds @param prefix as an extra data to the @param config. Returned
+	 * configuration object will contain an extra data named "configPrefix".
+	 * 
+	 * @param config
+	 *            {@link Configuration} object in which the configuration prefix
+	 *            need to be added.
+	 * @param prefix
+	 *            prefix that need to be added to the configuration.
+	 * @return Same @param config with configPrefix as an extra data.
+	 */
+	public static Configuration addConfigPrefix(Configuration config,
+			String prefix) {
+		Configuration.Builder builder = Configuration.builder(config);
+		builder.putExtraData("configPrefix", prefix);
+		return builder.build();
+	}
 }

From 6d137ae28e79f880c2071a1e014598a9005fa100 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 19:14:47 +0800
Subject: [PATCH 524/881] getConfigPrefix() added

---
 src/edu/mit/streamjit/util/ConfigurationUtils.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 4cb7bfaf..24019ce9 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -136,4 +136,16 @@ public static Configuration addConfigPrefix(Configuration config,
 		builder.putExtraData("configPrefix", prefix);
 		return builder.build();
 	}
+
+	/**
+	 * Gets configuration's prefix name from the configuration and returns.
+	 * 
+	 * @param config
+	 * @return prefix name of the configuration if exists. <code>null</code>
+	 *         otherwise.
+	 */
+	public static String getConfigPrefix(Configuration config) {
+		String prefix = (String) config.getExtraData("configPrefix");
+		return prefix == null ? "" : prefix;
+	}
 }

From cd048712ec875864d46a4ca924f1dfea74949f9d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 20:37:46 +0800
Subject: [PATCH 525/881] createDir() added.

---
 .../impl/distributed/common/Utils.java        | 35 ++++++++++++++-----
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index a7786313..bac8cbda 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -63,15 +63,32 @@ public static void printOutOfMemory() {
 				.println("Memory Use :" + usedMemory + "M/" + maxMemory + "M");
 	}
 
-	public static void createAppDir(String appName) {
-		File appDir = new File(appName);
-		if (appDir.exists()) {
-			if (appDir.isDirectory())
-				return;
-			else
-				System.err.println("A file exists in the name of appDir-"
-						+ appName);
+	/**
+	 * @param name
+	 *            name of the directory.
+	 * @return <code>true</code> if and only if the directory was created; false
+	 *         otherwise.
+	 */
+	private static boolean createDir(String name) {
+		File dir = new File(name);
+		if (dir.exists()) {
+			if (dir.isDirectory())
+				return true;
+			else {
+				System.err.println("A file exists in the name of dir-" + name);
+				return false;
+			}
 		} else
-			appDir.mkdir();
+			return dir.mkdir();
+	}
+
+	/**
+	 * @param name
+	 *            name of the directory.
+	 * @return <code>true</code> if and only if the directory was created; false
+	 *         otherwise.
+	 */
+	public static boolean createAppDir(String appName) {
+		return createDir(appName);
 	}
 }

From 1bf1ad87990045e640d2f66d1eb7a145eefcf69f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 20:48:58 +0800
Subject: [PATCH 526/881] createAppDir() creates a subdir "configurations"

---
 .../streamjit/impl/distributed/common/Utils.java  | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index bac8cbda..16d23439 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -79,16 +79,23 @@ private static boolean createDir(String name) {
 				return false;
 			}
 		} else
-			return dir.mkdir();
+			return dir.mkdirs();
 	}
 
 	/**
+	 * Creates app directory with the name of appName, and creates a sub
+	 * directory "configurations".
+	 * 
 	 * @param name
 	 *            name of the directory.
-	 * @return <code>true</code> if and only if the directory was created; false
-	 *         otherwise.
+	 * @return <code>true</code> if and only if the directories were created;
+	 *         false otherwise.
 	 */
 	public static boolean createAppDir(String appName) {
-		return createDir(appName);
+		if (createDir(appName))
+			return createDir(String.format("%s%sconfigurations", appName,
+					File.separator));
+		else
+			return false;
 	}
 }

From 8f66978186d2f790e7e36d7448ce32f873994f1d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 21:12:59 +0800
Subject: [PATCH 527/881] Saves each configurations's blobGraph.png

---
 .../mit/streamjit/impl/distributed/Visualizer.java  | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 979344c3..f8fbafa7 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -22,6 +22,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
  * Interface to visualize a stream graph and it's configurations. Use the
@@ -79,6 +80,11 @@ public static class DotVisualizer implements Visualizer {
 
 		private final String appName;
 
+		/**
+		 * namePrefix of the current configuration.
+		 */
+		private String namePrefix = "";
+
 		/**
 		 * Tells whether the dot tool is installed in the system or not.
 		 */
@@ -213,14 +219,15 @@ private void updateDot(Worker<?, ?> w) {
 
 		@Override
 		public void newConfiguration(Configuration cfg) {
-
+			namePrefix = ConfigurationUtils.getConfigPrefix(cfg);
 		}
 
 		private void runDot(String file) {
 			String fileName = String.format("./%s%s%s.dot", appName,
 					File.separator, file);
-			String outFileName = String.format("./%s%s%s.png", appName,
-					File.separator, file);
+			String outFileName = String.format(
+					"./%s%sconfigurations%s%s_%s.png", appName, File.separator,
+					File.separator, namePrefix, file);
 			ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng", fileName,
 					"-o", outFileName);
 			try {

From dd8480a4bb6d34375f6b30ca8b5bc2b5f9104e75 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 21:17:11 +0800
Subject: [PATCH 528/881] Default multiplier=50.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ac8e448c..de33c4fc 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -232,7 +232,7 @@ private boolean intermediateDraining() throws InterruptedException {
 	}
 
 	private int getMultiplier(Configuration config) {
-		int multiplier = 1000;
+		int multiplier = 50;
 		IntParameter mulParam = config.getParameter("multiplier",
 				IntParameter.class);
 		if (mulParam != null)

From 71c23821f8e26617e8ca39f8faa6c24a67160a4d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 22:28:29 +0800
Subject: [PATCH 529/881] CfgManager removed from GraphPropertyPrognosticator

---
 .../distributed/runtimer/GraphPropertyPrognosticator.java    | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 7aad65f5..1cd7afd4 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -24,8 +24,6 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	private final StreamJitApp<?, ?> app;
 
-	private final ConfigurationManager cfgManager;
-
 	private final FileWriter writer;
 
 	private int count = 0;
@@ -33,7 +31,6 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
 			ConfigurationManager cfgManager) {
 		this.app = app;
-		this.cfgManager = cfgManager;
 		this.writer = fileWriter();
 		writeHeader(writer);
 	}
@@ -41,8 +38,6 @@ public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
 	@Override
 	public boolean prognosticate(Configuration config) {
 		count++;
-		if (!cfgManager.newConfiguration(config))
-			return false;
 		float bigToSmallBlobRatio = bigToSmallBlobRatio();
 		float loadRatio = loadRatio();
 		float blobToNodeRatio = blobToNodeRatio();

From 633890dbeb5e003607f49eaa7f8bd150e0a5c50b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 8 Jan 2015 22:35:29 +0800
Subject: [PATCH 530/881] reconfigure() calls prognosticate(). Not tune().

---
 .../distributed/runtimer/OnlineTuner.java     | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index de33c4fc..e0cde36e 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -93,18 +93,14 @@ private void tune() {
 					ConfigurationUtils.saveConfg(cfgJson,
 							new Integer(round).toString(), app.name);
 
-				boolean possibleBetter = prognosticator.prognosticate(config);
-				if (possibleBetter) {
-					ret = reconfigure(config);
-					if (ret.first) {
-						prognosticator.time(ret.second);
-						tuner.writeLine(new Double(ret.second).toString());
-					} else {
-						tuner.writeLine("exit");
-						break;
-					}
-				} else
-					tuner.writeLine(new Double(-1).toString());
+				ret = reconfigure(config);
+				if (ret.first) {
+					prognosticator.time(ret.second);
+					tuner.writeLine(new Double(ret.second).toString());
+				} else {
+					tuner.writeLine("exit");
+					break;
+				}
 			}
 
 		} catch (IOException e) {
@@ -194,6 +190,9 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 			if (!cfgManager.newConfiguration(config))
 				return new Pair<Boolean, Long>(true, -1l);
 
+			if (!prognosticator.prognosticate(config))
+				return new Pair<Boolean, Long>(true, -1l);
+
 			if (!intermediateDraining())
 				return new Pair<Boolean, Long>(false, -1l);
 

From 6dbbcba923036d656101ab91dbfb7cda96fcac1b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 07:39:06 +0800
Subject: [PATCH 531/881] New Ant target runStreamNode added.

---
 jarapp.xml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index 463e4ab3..1b4fdb53 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -71,6 +71,17 @@
 		</exec>
 	</target>
 
+	<target name="runStreamNode">
+		<exec dir="." executable="gnome-terminal" osfamily="unix">
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar StreamNode.jar" />
+		</exec>
+	</target>
+
 
 	<target name="getLogin">
 		<input message="Enter Username:" addproperty="username" />

From f01f85dc388cf0e730e4b3d15215e62fb2017200 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 07:45:07 +0800
Subject: [PATCH 532/881] Ant script: run -> runAll

---
 jarapp.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index 1b4fdb53..7ba19229 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -54,10 +54,10 @@
 		</scp>
 	</target>
 
-	<target name="buildAndRun" depends="buildApp, run">
+	<target name="buildAndRun" depends="buildApp, runAll">
 	</target>
 
-	<target name="run">
+	<target name="runAll">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
 			<arg value="--tab" />
 			<arg value="-e" />

From 881713d4a974384cddd7e33a871c2d75fb04f8cf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 07:51:20 +0800
Subject: [PATCH 533/881] Ant Script:Desctiptions added to Ant targets

---
 jarapp.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index 7ba19229..64e40687 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -57,7 +57,7 @@
 	<target name="buildAndRun" depends="buildApp, runAll">
 	</target>
 
-	<target name="runAll">
+	<target name="runAll" description="Runs app.jar and two instances of StreamNode.jar">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
 			<arg value="--tab" />
 			<arg value="-e" />
@@ -71,7 +71,7 @@
 		</exec>
 	</target>
 
-	<target name="runStreamNode">
+	<target name="runStreamNode" description="Runs two instances of StreamNode.jar">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
 			<arg value="--tab" />
 			<arg value="-e" />

From 1f7a96c97423968ea222f05f69e0d5f175c49f32 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 07:54:28 +0800
Subject: [PATCH 534/881] Ant Script: StreamNode->buildStreamNode

---
 jarapp.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index 64e40687..0f7a08b9 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -11,7 +11,7 @@
 	<!-- property name="username" value="sumanan" /-->
 	<!-- property name="password" value="" / -->
 
-	<target name="buildApp" depends="StreamNode" description="Exports ${app}.jar">
+	<target name="buildApp" depends="buildStreamNode" description="Exports ${app}.jar">
 		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
 				<attribute name="Main-Class" value="${mainclass}" />
@@ -29,7 +29,7 @@
 		</jar>
 	</target>
 
-	<target name="StreamNode" description="Exports StreamNode.jar">
+	<target name="buildStreamNode" description="Exports StreamNode.jar">
 		<jar destfile="StreamNode.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
 				<attribute name="Main-Class" value="edu.mit.streamjit.impl.distributed.node.StreamNode" />

From b7df8c31b0309884a641fa11c682f5a95bc3cf82 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 07:56:57 +0800
Subject: [PATCH 535/881] Ant Script:New target buildAll has been added.

---
 jarapp.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index 0f7a08b9..91501abd 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -11,7 +11,7 @@
 	<!-- property name="username" value="sumanan" /-->
 	<!-- property name="password" value="" / -->
 
-	<target name="buildApp" depends="buildStreamNode" description="Exports ${app}.jar">
+	<target name="buildApp" description="Exports ${app}.jar">
 		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
 				<attribute name="Main-Class" value="${mainclass}" />
@@ -57,6 +57,9 @@
 	<target name="buildAndRun" depends="buildApp, runAll">
 	</target>
 
+	<target name="buildAll" depends="buildApp, buildStreamNode">
+	</target>
+
 	<target name="runAll" description="Runs app.jar and two instances of StreamNode.jar">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
 			<arg value="--tab" />

From 377c5c1a3faa6e3883db949762fadc8b254d139b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 08:04:08 +0800
Subject: [PATCH 536/881] Other targets call buildAll. Not buildApp.

---
 jarapp.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index 91501abd..22f461ea 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -47,14 +47,14 @@
 		</jar>
 	</target>
 
-	<target name="sendtoLanka" depends="buildApp, getLogin">
+	<target name="sendtoLanka" depends="buildAll, getLogin">
 		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
 			<fileset file="StreamNode.jar" />
 			<fileset file="${app}.jar" />
 		</scp>
 	</target>
 
-	<target name="buildAndRun" depends="buildApp, runAll">
+	<target name="buildAndRunAll" depends="buildAll, runAll">
 	</target>
 
 	<target name="buildAll" depends="buildApp, buildStreamNode">

From b55159b64b0a690b61de391bacaf79f247bd44c5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 08:10:37 +0800
Subject: [PATCH 537/881] Ant Script: targets are logically ordered.

---
 jarapp.xml | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index 22f461ea..db5da668 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -47,24 +47,11 @@
 		</jar>
 	</target>
 
-	<target name="sendtoLanka" depends="buildAll, getLogin">
-		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
-			<fileset file="StreamNode.jar" />
-			<fileset file="${app}.jar" />
-		</scp>
-	</target>
-
-	<target name="buildAndRunAll" depends="buildAll, runAll">
-	</target>
-
 	<target name="buildAll" depends="buildApp, buildStreamNode">
 	</target>
 
-	<target name="runAll" description="Runs app.jar and two instances of StreamNode.jar">
+	<target name="runStreamNode" description="Runs two instances of StreamNode.jar">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
-			<arg value="--tab" />
-			<arg value="-e" />
-			<arg value="java -jar ${app}.jar" />
 			<arg value="--tab" />
 			<arg value="-e" />
 			<arg value="java -jar StreamNode.jar" />
@@ -74,8 +61,11 @@
 		</exec>
 	</target>
 
-	<target name="runStreamNode" description="Runs two instances of StreamNode.jar">
+	<target name="runAll" description="Runs app.jar and two instances of StreamNode.jar">
 		<exec dir="." executable="gnome-terminal" osfamily="unix">
+			<arg value="--tab" />
+			<arg value="-e" />
+			<arg value="java -jar ${app}.jar" />
 			<arg value="--tab" />
 			<arg value="-e" />
 			<arg value="java -jar StreamNode.jar" />
@@ -85,6 +75,15 @@
 		</exec>
 	</target>
 
+	<target name="buildAndRunAll" depends="buildAll, runAll">
+	</target>
+
+	<target name="sendtoLanka" depends="buildAll, getLogin">
+		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
+			<fileset file="StreamNode.jar" />
+			<fileset file="${app}.jar" />
+		</scp>
+	</target>
 
 	<target name="getLogin">
 		<input message="Enter Username:" addproperty="username" />

From 50a6ee4dd54c7fcf61033a15ec15772923577f9f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 08:12:12 +0800
Subject: [PATCH 538/881] Ant Script: Default target buildApp->buildAll

---
 jarapp.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index db5da668..eb251fb1 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!--Sumanan, sumanan@mit.edu, 15-12-2014 -->
-<project default="buildApp" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
+<project default="buildAll" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
 	<property file="jarapp.properties" />
 	<!--property name="app" value="NestedSplitJoin" /-->
 	<!--property name="mainclass" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->

From f6249309a0bf23fdbf425df9460e83ca8686f1b8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 08:26:55 +0800
Subject: [PATCH 539/881] AntScript: build->export

The word 'build' has been replaced the word 'export'. Because the Ant targets
actually exports the already built *.class files to *.jar files. They do not
build(compile) *.java files to *.class files.
---
 jarapp.xml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index eb251fb1..694eb460 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!--Sumanan, sumanan@mit.edu, 15-12-2014 -->
-<project default="buildAll" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
+<project default="exportAll" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
 	<property file="jarapp.properties" />
 	<!--property name="app" value="NestedSplitJoin" /-->
 	<!--property name="mainclass" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->
@@ -11,7 +11,7 @@
 	<!-- property name="username" value="sumanan" /-->
 	<!-- property name="password" value="" / -->
 
-	<target name="buildApp" description="Exports ${app}.jar">
+	<target name="exportApp.jar" description="Exports ${app}.jar">
 		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
 				<attribute name="Main-Class" value="${mainclass}" />
@@ -29,7 +29,7 @@
 		</jar>
 	</target>
 
-	<target name="buildStreamNode" description="Exports StreamNode.jar">
+	<target name="exportStreamNode.jar" description="Exports StreamNode.jar">
 		<jar destfile="StreamNode.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
 				<attribute name="Main-Class" value="edu.mit.streamjit.impl.distributed.node.StreamNode" />
@@ -47,7 +47,7 @@
 		</jar>
 	</target>
 
-	<target name="buildAll" depends="buildApp, buildStreamNode">
+	<target name="exportAll" depends="exportApp.jar, exportStreamNode.jar">
 	</target>
 
 	<target name="runStreamNode" description="Runs two instances of StreamNode.jar">
@@ -75,10 +75,10 @@
 		</exec>
 	</target>
 
-	<target name="buildAndRunAll" depends="buildAll, runAll">
+	<target name="exportAndRunAll" depends="exportAll, runAll">
 	</target>
 
-	<target name="sendtoLanka" depends="buildAll, getLogin">
+	<target name="sendtoLanka" depends="exportAll, getLogin">
 		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
 			<fileset file="StreamNode.jar" />
 			<fileset file="${app}.jar" />

From 126b062428c5b3c997e268a0d869c0e81c001dd6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 23:09:34 +0800
Subject: [PATCH 540/881] verifyTuningTimes()'s null check removed.

evaluateConfig() checks the null condition and writes the status in to the
file.
---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java    | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e0cde36e..69c5e2f2 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -167,8 +167,6 @@ private void verifyTuningTimes() {
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
 					new Integer(n).toString());
 
-			if (cfg == null)
-				continue;
 			evaluateConfig(cfg, cfgName);
 		}
 		terminate();

From d8f40e36d0a4c6b9a66ebd3869af7f75888a69f2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 23:16:35 +0800
Subject: [PATCH 541/881] verifyTuningTimes() uses cfgPrefixes.

Previously it used cfg no.
---
 .../impl/distributed/runtimer/OnlineTuner.java       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 69c5e2f2..78924c43 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -160,13 +160,13 @@ private void evaluate() {
 	 * This method can be called after the completion of the tuning.
 	 */
 	private void verifyTuningTimes() {
-		int[] cfgNos = { 10, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500,
-				550, 600, 650, 700, 750, 800, 850, 900, 950, 1000 };
-		for (int n : cfgNos) {
-			String cfgName = String.format("%d_%s.cfg", n, app.name);
+		String[] cfgPrefixes = { "10", "50", "100", "150", "200", "250", "300",
+				"350", "400", "450", "500", "550", "600", "650", "700", "750",
+				"800", "850", "900", "950", "1000" };
+		for (String prefix : cfgPrefixes) {
+			String cfgName = String.format("%s_%s.cfg", prefix, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
-					new Integer(n).toString());
-
+					prefix);
 			evaluateConfig(cfg, cfgName);
 		}
 		terminate();

From a75abafa1a02613791cac87603ace70827ca811f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 23:18:36 +0800
Subject: [PATCH 542/881] hand & final prefixes added to verifyTuningTimes()

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 78924c43..20f99c6b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -162,7 +162,7 @@ private void evaluate() {
 	private void verifyTuningTimes() {
 		String[] cfgPrefixes = { "10", "50", "100", "150", "200", "250", "300",
 				"350", "400", "450", "500", "550", "600", "650", "700", "750",
-				"800", "850", "900", "950", "1000" };
+				"800", "850", "900", "950", "1000", "hand", "final" };
 		for (String prefix : cfgPrefixes) {
 			String cfgName = String.format("%s_%s.cfg", prefix, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,

From c675f4838a8ff7c79838419fbbf00ad3760494e9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 10 Jan 2015 23:44:25 +0800
Subject: [PATCH 543/881] evaluateConfig() updates prognosticator.time()

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 20f99c6b..76da57c5 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -300,6 +300,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 				for (int i = 0; i < count; i++) {
 					ret = reconfigure(cfg);
 					if (ret.first) {
+						prognosticator.time(ret.second);
 						writer.write(ret.second.toString());
 						writer.write('\n');
 						writer.flush();

From c8b3cba2734a62fb32191d6af3eddb666e3eb569 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 11 Jan 2015 10:49:32 +0800
Subject: [PATCH 544/881] changeMultiplierVal() takes arguments.

Previously these arguments were hard coded.
changeMultiplierVal() -> changeMultiplierVal(String appName, String namePrefix).
---
 .../mit/streamjit/impl/common/ConfigurationEditor.java   | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index cdc7fafe..9e75bf37 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -37,18 +37,15 @@ public class ConfigurationEditor {
 	public static void main(String[] args) throws IOException {
 		Pair<String, Integer> ret = generate(
 				new ChannelVocoder7.ChannelVocoder7Kernel(), 16);
-		// edit1(name, noofwrks);
+		// edit1("FilterBankPipeline", "463", 83);
 		// print("4366_NestedSplitJoinCore.cfg");
-		// convert();
-		// changeMultiplierVal();
+		// changeMultiplierVal("NestedSplitJoinCore","final");
 	}
 
 	/**
 	 * Reads a configuration and changes its multiplier value.
 	 */
-	private static void changeMultiplierVal() {
-		String appName = "NestedSplitJoinCore";
-		String namePrefix = "final_";
+	private static void changeMultiplierVal(String appName, String namePrefix) {
 		Configuration config = ConfigurationUtils.readConfiguration(appName,
 				namePrefix);
 		if (config == null)

From d72509bfe49f82e52756016bad61d67cde247d89 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 13 Jan 2015 19:52:48 +0800
Subject: [PATCH 545/881] StreamPathBuilder added.

StreamPathBuilder generate all stream paths in a stream graph.
---
 .../impl/distributed/StreamPathBuilder.java   | 163 ++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java b/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java
new file mode 100644
index 00000000..90ad6eca
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java
@@ -0,0 +1,163 @@
+package edu.mit.streamjit.impl.distributed;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import edu.mit.streamjit.api.Filter;
+import edu.mit.streamjit.api.Joiner;
+import edu.mit.streamjit.api.OneToOneElement;
+import edu.mit.streamjit.api.Pipeline;
+import edu.mit.streamjit.api.Splitjoin;
+import edu.mit.streamjit.api.Splitter;
+import edu.mit.streamjit.api.StreamVisitor;
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
+import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.test.apps.filterbank6.FilterBank6;
+
+/**
+ * Generate all stream paths in a stream graph.
+ * 
+ * @author sumanan
+ * @since 13 Jan, 2015
+ */
+public class StreamPathBuilder {
+
+	/**
+	 * streamGraph must be connected before requesting for paths. Use
+	 * {@link ConnectWorkersVisitor} to connect the streamGraph.
+	 * 
+	 * @param streamGraph
+	 * @return Set of all paths in the streamGraph.
+	 */
+	public static Set<List<Integer>> paths(OneToOneElement<?, ?> streamGraph) {
+		PathVisitor v = new PathVisitor();
+		streamGraph.visit(v);
+		return v.currentUnfinishedPathSet;
+	}
+
+	private static class PathVisitor extends StreamVisitor {
+
+		/**
+		 * Paths those are currently being built. These paths will get extended
+		 * as StreamPathBuilder visits through the stream graph.
+		 */
+		private Set<List<Integer>> currentUnfinishedPathSet;
+
+		/**
+		 * Keeps track of paths to all {@link Splitter} encountered in a stack.
+		 * Once corresponding {@link Joiner} is visited, path set will be popped
+		 * from this stack.
+		 */
+		private Deque<Set<List<Integer>>> pathToSplitterStack;
+
+		/**
+		 * Unfinished path sets which are waiting for a correct joiner. Path set
+		 * will be popped from this stack once correct joiner is reached.
+		 */
+		private Deque<Set<List<Integer>>> waitingForJoinerStack;
+
+		private PathVisitor() {
+			currentUnfinishedPathSet = new HashSet<>();
+			pathToSplitterStack = new ArrayDeque<>();
+			waitingForJoinerStack = new ArrayDeque<>();
+		}
+
+		@Override
+		public void beginVisit() {
+			currentUnfinishedPathSet.clear();
+			pathToSplitterStack.clear();
+			waitingForJoinerStack.clear();
+			currentUnfinishedPathSet.add(new LinkedList<Integer>());
+		}
+
+		@Override
+		public void visitFilter(Filter<?, ?> filter) {
+			addToCurrentPath(filter);
+		}
+
+		@Override
+		public boolean enterPipeline(Pipeline<?, ?> pipeline) {
+			return true;
+		}
+
+		@Override
+		public void exitPipeline(Pipeline<?, ?> pipeline) {
+		}
+
+		@Override
+		public boolean enterSplitjoin(Splitjoin<?, ?> splitjoin) {
+			return true;
+		}
+
+		@Override
+		public void visitSplitter(Splitter<?, ?> splitter) {
+			addToCurrentPath(splitter);
+			pathToSplitterStack.push(currentUnfinishedPathSet);
+			waitingForJoinerStack.push(new HashSet<List<Integer>>());
+		}
+
+		@Override
+		public boolean enterSplitjoinBranch(OneToOneElement<?, ?> element) {
+			currentUnfinishedPathSet = new HashSet<List<Integer>>();
+			for (List<Integer> splitterPath : pathToSplitterStack.peek()) {
+				currentUnfinishedPathSet.add(new LinkedList<Integer>(
+						splitterPath));
+			}
+			return true;
+		}
+
+		@Override
+		public void exitSplitjoinBranch(OneToOneElement<?, ?> element) {
+			waitingForJoinerStack.peek().addAll(currentUnfinishedPathSet);
+		}
+
+		@Override
+		public void visitJoiner(Joiner<?, ?> joiner) {
+			currentUnfinishedPathSet = waitingForJoinerStack.pop();
+			addToCurrentPath(joiner);
+			pathToSplitterStack.pop();
+		}
+
+		@Override
+		public void exitSplitjoin(Splitjoin<?, ?> splitjoin) {
+		}
+
+		@Override
+		public void endVisit() {
+			if (!waitingForJoinerStack.isEmpty())
+				throw new IllegalStateException("waitingForJoiner not empty");
+			if (!pathToSplitterStack.isEmpty())
+				throw new IllegalStateException("pathToSplitter not empty");
+			for (List<Integer> path : currentUnfinishedPathSet) {
+				for (Integer i : path) {
+					System.out.print(i.toString() + "->");
+				}
+				System.out.println();
+			}
+		}
+
+		/**
+		 * Extends all current unfinished path set with the {@link Worker} w.
+		 * 
+		 * @param w
+		 */
+		private void addToCurrentPath(Worker<?, ?> w) {
+			int id = Workers.getIdentifier(w);
+			for (List<Integer> path : currentUnfinishedPathSet)
+				path.add(id);
+		}
+	}
+
+	public static void main(String[] args) {
+		OneToOneElement<?, ?> stream = new FilterBank6.FilterBankPipeline();
+		new StreamJitApp<>(stream); // Connects the stream into stream graph.
+		StreamPathBuilder.paths(stream);
+	}
+}
\ No newline at end of file

From e4bc0c34e2c28f3a87bab70c7e8cea098dc6eab1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 13 Jan 2015 20:03:52 +0800
Subject: [PATCH 546/881] paths() method added to StreamJitApp.

---
 .../mit/streamjit/impl/distributed/StreamJitApp.java  | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 74b6b443..0751692b 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -329,4 +329,15 @@ private List<MessageConstraint> getConstrains() {
 			Portals.setConstraints(portal, constraints);
 		return constraints;
 	}
+
+	/**
+	 * Uses {@link StreamPathBuilder} to generate all paths in the streamGraph
+	 * of this {@link StreamJitApp}. Check {@link StreamPathBuilder} for more
+	 * information.
+	 * 
+	 * @return Set of all paths in the streamGraph of this {@link StreamJitApp}.
+	 */
+	public Set<List<Integer>> paths() {
+		return StreamPathBuilder.paths(streamGraph);
+	}
 }

From 0c973a7479c7f6e5bac6b8733113de93d8236d88 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 13 Jan 2015 20:06:14 +0800
Subject: [PATCH 547/881] hasCycle() method added.

GraphPropertyPrognosticator builds machine path for each stream path and checks
for cycles in the machine path. My finding was if cycles exists in a machine
path, then that configuration is most likely to perform bad.
---
 .../runtimer/GraphPropertyPrognosticator.java | 59 ++++++++++++++++++-
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 1cd7afd4..ffdceb65 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -4,6 +4,7 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -28,11 +29,14 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	private int count = 0;
 
+	private final Set<List<Integer>> paths;
+
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
 			ConfigurationManager cfgManager) {
 		this.app = app;
 		this.writer = fileWriter();
 		writeHeader(writer);
+		paths = app.paths();
 	}
 
 	@Override
@@ -42,13 +46,14 @@ public boolean prognosticate(Configuration config) {
 		float loadRatio = loadRatio();
 		float blobToNodeRatio = blobToNodeRatio();
 		float BoundaryChannelRatio = totalToBoundaryChannelRatio();
+		boolean hasCycle = hasCycle();
 		try {
 			writer.write(String.format("\n%4d\t\t", count));
 			writer.write(String.format("%.2f\t\t", bigToSmallBlobRatio));
 			writer.write(String.format("%.2f\t\t", loadRatio));
 			writer.write(String.format("%.2f\t\t", blobToNodeRatio));
 			writer.write(String.format("%.2f\t\t", BoundaryChannelRatio));
-			writer.flush();
+			writer.write(String.format("%s\t\t", hasCycle ? "True" : "False"));
 		} catch (Exception e) {
 			e.printStackTrace();
 		}
@@ -164,6 +169,8 @@ private static void writeHeader(FileWriter writer) {
 			writer.write("\t\t");
 			writer.write(String.format("%.7s", "BoundaryChannelRatio"));
 			writer.write("\t\t");
+			writer.write(String.format("%.7s", "hasCycles"));
+			writer.write("\t\t");
 			writer.write(String.format("%.7s", "time"));
 			writer.write("\t\t");
 			writer.flush();
@@ -181,4 +188,52 @@ public void time(double time) {
 			e.printStackTrace();
 		}
 	}
-}
\ No newline at end of file
+
+	private boolean hasCycle() {
+		Set<List<Integer>> machinePaths = buildMachinePaths();
+		for (List<Integer> path : machinePaths) {
+			Set<Integer> machines = new HashSet<Integer>();
+			for (int i = 0; i < path.size() - 1; i++) {
+				int machine = path.get(i);
+				if (machines.contains(machine))
+					return true;
+				machines.add(machine);
+
+			}
+		}
+		return false;
+	}
+
+	private Set<List<Integer>> buildMachinePaths() {
+		Set<List<Integer>> machinePaths = new HashSet<List<Integer>>();
+		List<Integer> machinePath;
+		for (List<Integer> path : paths) {
+			machinePath = new LinkedList<Integer>();
+			int curMachine = -1;
+			for (Integer worker : path) {
+				int machine = getAssignedMachine(worker);
+				if (curMachine != machine) {
+					machinePath.add(machine);
+					curMachine = machine;
+				}
+			}
+			machinePaths.add(machinePath);
+		}
+		return machinePaths;
+	}
+
+	private int getAssignedMachine(int workerID) {
+		for (Integer machineID : app.partitionsMachineMap.keySet()) {
+			for (Set<Worker<?, ?>> blobWorkers : app.partitionsMachineMap
+					.get(machineID)) {
+				for (Worker<?, ?> w : blobWorkers) {
+					if (Workers.getIdentifier(w) == workerID)
+						return machineID;
+				}
+			}
+		}
+
+		throw new IllegalArgumentException(String.format(
+				"Worker-%d is not assigned to anyof the machines", workerID));
+	}
+}

From b1faa5660fceb35c12bf6b914a4d3d2e122fc386 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 13 Jan 2015 20:31:07 +0800
Subject: [PATCH 548/881] Uses cfgPrefix as primary key

---
 .../distributed/runtimer/GraphPropertyPrognosticator.java  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index ffdceb65..a11a3d26 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -14,6 +14,7 @@
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
  * Assumes the cluster environment is homogeneous.
@@ -27,8 +28,6 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	private final FileWriter writer;
 
-	private int count = 0;
-
 	private final Set<List<Integer>> paths;
 
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
@@ -41,14 +40,14 @@ public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
 
 	@Override
 	public boolean prognosticate(Configuration config) {
-		count++;
+		String cfgPrefix = ConfigurationUtils.getConfigPrefix(config);
 		float bigToSmallBlobRatio = bigToSmallBlobRatio();
 		float loadRatio = loadRatio();
 		float blobToNodeRatio = blobToNodeRatio();
 		float BoundaryChannelRatio = totalToBoundaryChannelRatio();
 		boolean hasCycle = hasCycle();
 		try {
-			writer.write(String.format("\n%4d\t\t", count));
+			writer.write(String.format("\n%6s\t\t", cfgPrefix));
 			writer.write(String.format("%.2f\t\t", bigToSmallBlobRatio));
 			writer.write(String.format("%.2f\t\t", loadRatio));
 			writer.write(String.format("%.2f\t\t", blobToNodeRatio));

From 6ee36be6d30f531a262fd639bba90c7015cf51da Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 13 Jan 2015 20:33:13 +0800
Subject: [PATCH 549/881] Unused param removed.

Parameter configurationManager is removed from GraphPropertyPrognosticator's
constructor.
---
 .../distributed/runtimer/GraphPropertyPrognosticator.java     | 4 +---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index a11a3d26..1947a009 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -11,7 +11,6 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.util.ConfigurationUtils;
@@ -30,8 +29,7 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	private final Set<List<Integer>> paths;
 
-	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app,
-			ConfigurationManager cfgManager) {
+	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app) {
 		this.app = app;
 		this.writer = fileWriter();
 		writeHeader(writer);
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 76da57c5..2083a90d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -45,7 +45,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
 		this.logger = logger;
-		this.prognosticator = new GraphPropertyPrognosticator(app, cfgManager);
+		this.prognosticator = new GraphPropertyPrognosticator(app);
 	}
 
 	@Override

From 9b7ff4d5a5b29c07601018635151877cd118b9fd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 17 Jan 2015 10:22:51 +0800
Subject: [PATCH 550/881] Unnecessary print msgs removed.

---
 .../impl/distributed/runtimer/GraphPropertyPrognosticator.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 1947a009..bca64a19 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -74,7 +74,6 @@ private float bigToSmallBlobRatio() {
 			}
 		}
 		float blobRatio = ((float) max) / min;
-		System.out.println("blobRatio - " + blobRatio);
 		return blobRatio;
 	}
 
@@ -96,7 +95,6 @@ private float loadRatio() {
 			max = Math.max(max, workersInCurrentNode);
 		}
 		float loadRatio = ((float) max) / min;
-		System.out.println("loadRatio - " + loadRatio);
 		return loadRatio;
 	}
 

From 52468f5e6850f102ca56b8182274ce62ba6bccf6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 18 Jan 2015 14:45:08 +0800
Subject: [PATCH 551/881] Refactored:printPaths() added.

---
 .../impl/distributed/StreamPathBuilder.java   | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java b/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java
index 90ad6eca..dbef4b31 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamPathBuilder.java
@@ -135,12 +135,7 @@ public void endVisit() {
 				throw new IllegalStateException("waitingForJoiner not empty");
 			if (!pathToSplitterStack.isEmpty())
 				throw new IllegalStateException("pathToSplitter not empty");
-			for (List<Integer> path : currentUnfinishedPathSet) {
-				for (Integer i : path) {
-					System.out.print(i.toString() + "->");
-				}
-				System.out.println();
-			}
+			// printPaths();
 		}
 
 		/**
@@ -153,6 +148,18 @@ private void addToCurrentPath(Worker<?, ?> w) {
 			for (List<Integer> path : currentUnfinishedPathSet)
 				path.add(id);
 		}
+
+		/**
+		 * Prints all paths in the stream graph.
+		 */
+		private void printPaths() {
+			for (List<Integer> path : currentUnfinishedPathSet) {
+				for (Integer i : path) {
+					System.out.print(i.toString() + "->");
+				}
+				System.out.println();
+			}
+		}
 	}
 
 	public static void main(String[] args) {

From b4cbc945c554313717699d462b9b06eed84273b0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 18 Jan 2015 15:56:00 +0800
Subject: [PATCH 552/881] writeReadMeTxt() added.

---
 .../impl/distributed/StreamJitApp.java        |  1 +
 .../impl/distributed/common/Utils.java        | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 0751692b..b063234a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -103,6 +103,7 @@ public StreamJitApp(OneToOneElement<I, O> streamGraph) {
 				.getCodeSource().getLocation().getPath();
 		this.constraints = getConstrains();
 		Utils.createAppDir(name);
+		Utils.writeReadMeTxt(name);
 		visualizer = new Visualizer.DotVisualizer(streamGraph);
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 16d23439..fe1c07a7 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -1,9 +1,14 @@
 package edu.mit.streamjit.impl.distributed.common;
 
 import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
 import java.util.Collections;
 import java.util.Set;
 
@@ -98,4 +103,46 @@ public static boolean createAppDir(String appName) {
 		else
 			return false;
 	}
+
+	/**
+	 * Writes README.txt. Mainly saves GlobalConstant values.
+	 * 
+	 * @param appName
+	 */
+	public static void writeReadMeTxt(String appName) {
+		try {
+			FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
+					appName, File.separator));
+			DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+			Calendar cal = Calendar.getInstance();
+			writer.write(dateFormat.format(cal.getTime()) + "\n");
+			writer.write(appName + "\n");
+			update(writer, "tunerStartMode", GlobalConstants.tunerStartMode);
+			update(writer, "useDrainData", GlobalConstants.useDrainData);
+			update(writer, "needDrainDeadlockHandler",
+					GlobalConstants.needDrainDeadlockHandler);
+			update(writer, "tune", GlobalConstants.tune);
+			update(writer, "saveAllConfigurations",
+					GlobalConstants.saveAllConfigurations);
+			update(writer, "outputCount", GlobalConstants.outputCount);
+			update(writer, "useCompilerBlob", GlobalConstants.useCompilerBlob);
+			update(writer, "printOutputCountPeriod",
+					GlobalConstants.printOutputCountPeriod);
+			update(writer, "singleNodeOnline", GlobalConstants.singleNodeOnline);
+			update(writer, "maxNumCores", GlobalConstants.maxNumCores);
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private static void update(FileWriter writer, String name, int val)
+			throws IOException {
+		writer.write(String.format("%s=%d\n", name, val));
+	}
+
+	private static void update(FileWriter writer, String name, boolean val)
+			throws IOException {
+		writer.write(String.format("%s=%s\n", name, val ? "True" : "False"));
+	}
 }

From cb8eae15286ba0f5653193d8edc667e2b4978e49 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 17:37:32 +0800
Subject: [PATCH 553/881] Renamed: TailChannel->TailChannels

---
 .../impl/distributed/{TailChannel.java => TailChannels.java}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/{TailChannel.java => TailChannels.java} (100%)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
similarity index 100%
rename from src/edu/mit/streamjit/impl/distributed/TailChannel.java
rename to src/edu/mit/streamjit/impl/distributed/TailChannels.java

From cc8925ba6d0cd8cc943a93cf3aa71e31f0a2a758 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 17:40:18 +0800
Subject: [PATCH 554/881] Renamed:TailChannel->TailChannels

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 4 ++--
 src/edu/mit/streamjit/impl/distributed/TailChannels.java    | 4 ++--
 .../streamjit/impl/distributed/common/GlobalConstants.java  | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 130611ff..f98c17d3 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -87,7 +87,7 @@ public class StreamJitAppManager {
 	 * we need to pull the sink's output in to the {@link Controller} in order
 	 * to make {@link CompiledStream} .pull() to work.
 	 */
-	private TailChannel tailChannel;
+	private TailChannels tailChannel;
 
 	private Thread tailThread;
 
@@ -323,7 +323,7 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
-		tailChannel = new TailChannel(bufferMap.get(tailToken),
+		tailChannel = new TailChannels(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
 						+ tailToken.toString(), 0, skipCount,
 				GlobalConstants.outputCount, app.name);
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 29a245f7..c03ca525 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -18,7 +18,7 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
-public class TailChannel extends BlockingInputChannel {
+public class TailChannels extends BlockingInputChannel {
 
 	private final int skipCount;
 
@@ -64,7 +64,7 @@ public class TailChannel extends BlockingInputChannel {
 	 *            get this amount of outputs ( after skipping skipCount number
 	 *            of outputs at the beginning).
 	 */
-	public TailChannel(Buffer buffer, ConnectionProvider conProvider,
+	public TailChannels(Buffer buffer, ConnectionProvider conProvider,
 			ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 			int skipCount, int steadyCount, String appName) {
 		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 72225551..df4bf125 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -2,7 +2,7 @@
 
 import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
-import edu.mit.streamjit.impl.distributed.TailChannel;
+import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
 import edu.mit.streamjit.tuner.TCPTuner;
@@ -96,7 +96,7 @@ private GlobalConstants() {
 	 * Otherwise, just use the fixed configuration file to run the program. No
 	 * tuning, no intermediate draining. In this mode (tune = false), time taken
 	 * to pass fixed number of input will be measured for 30 rounds and logged
-	 * into FixedOutPut.txt. See {@link TailChannel} for the file logging
+	 * into FixedOutPut.txt. See {@link TailChannels} for the file logging
 	 * details.
 	 * <ol>
 	 * 0 - No tuning, uses configuration file to run.
@@ -128,7 +128,7 @@ private GlobalConstants() {
 	/**
 	 * Period to print output count periodically. This printing feature get
 	 * turned off if this value is less than 1. Time unit is ms. See
-	 * {@link TailChannel}.
+	 * {@link TailChannels}.
 	 */
 	public static final int printOutputCountPeriod = 6000;
 

From 11df670aa7f832ac9939b894db5f22db2f766409 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 18:43:48 +0800
Subject: [PATCH 555/881] reset() has been made private

TailChannel.reset().
---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index c03ca525..f5adcc83 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -156,7 +156,7 @@ public void run() {
 				GlobalConstants.printOutputCountPeriod, TimeUnit.MILLISECONDS);
 	}
 
-	public void reset() {
+	private void reset() {
 		count = 0;
 		lastCount = 0;
 		skipLatch.countDown();

From 569fb33beb2af5a2d2ff9e6894c20ef0b79e7536 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 18:46:01 +0800
Subject: [PATCH 556/881] Renamed: reset() -> resetAll()

---
 .../mit/streamjit/impl/distributed/TailChannels.java   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index f5adcc83..e1d9cd7e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -103,7 +103,7 @@ public void receiveData() {
 	public void stop(DrainType type) {
 		super.stop(type);
 		if (pLogger != null) {
-			reset();
+			resetAll();
 			pLogger.stopLogging();
 		}
 		if (scheduledExecutorService != null)
@@ -120,7 +120,7 @@ public void stop(DrainType type) {
 	 * @throws InterruptedException
 	 */
 	public long getFixedOutputTime() throws InterruptedException {
-		reset();
+		resetAll();
 		skipLatch.await();
 		Stopwatch stopwatch = Stopwatch.createStarted();
 		steadyLatch.await();
@@ -156,7 +156,7 @@ public void run() {
 				GlobalConstants.printOutputCountPeriod, TimeUnit.MILLISECONDS);
 	}
 
-	private void reset() {
+	private void resetAll() {
 		count = 0;
 		lastCount = 0;
 		skipLatch.countDown();
@@ -168,8 +168,8 @@ private void reset() {
 	}
 
 	/**
-	 * We need this method apart from {@link #reset()}, because the
-	 * {@link #reset()} method creates the latches immediately after
+	 * We need this method apart from {@link #resetAll()}, because the
+	 * {@link #resetAll()} method creates the latches immediately after
 	 * countDown(). This causes the threads which are waiting on the latches at
 	 * {@link #getFixedOutputTime()} will not be released properly.
 	 */

From 6f887195fb9da6f62483640d249b68665bda16fc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 18:48:27 +0800
Subject: [PATCH 557/881] Renamed:releaseAll()->reset()

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java   | 4 ++--
 src/edu/mit/streamjit/impl/distributed/TailChannels.java      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index f98c17d3..77140cd5 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -272,7 +272,7 @@ public void setDrainer(AbstractDrainer drainer) {
 
 	public void stop() {
 		this.status = AppStatus.STOPPED;
-		tailChannel.releaseAll();
+		tailChannel.reset();
 		controller.closeAll();
 		dp.drainer.stop();
 	}
@@ -388,7 +388,7 @@ public void processERROR() {
 			this.error = true;
 			// This will release the OpenTuner thread which is waiting for fixed
 			// output.
-			tailChannel.releaseAll();
+			tailChannel.reset();
 		}
 
 		@Override
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index e1d9cd7e..f3e99413 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -173,7 +173,7 @@ private void resetAll() {
 	 * countDown(). This causes the threads which are waiting on the latches at
 	 * {@link #getFixedOutputTime()} will not be released properly.
 	 */
-	public void releaseAll() {
+	public void reset() {
 		steadyLatch.countDown();
 		skipLatch.countDown();
 		count = 0;

From a548f086a4893559dd31f002bf5ac033dc91c281 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 18:52:32 +0800
Subject: [PATCH 558/881] Renamed:resetAll()->releaseAndInitilize()

---
 .../streamjit/impl/distributed/TailChannels.java  | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index f3e99413..386af0d8 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -103,7 +103,7 @@ public void receiveData() {
 	public void stop(DrainType type) {
 		super.stop(type);
 		if (pLogger != null) {
-			resetAll();
+			releaseAndInitilize();
 			pLogger.stopLogging();
 		}
 		if (scheduledExecutorService != null)
@@ -120,7 +120,7 @@ public void stop(DrainType type) {
 	 * @throws InterruptedException
 	 */
 	public long getFixedOutputTime() throws InterruptedException {
-		resetAll();
+		releaseAndInitilize();
 		skipLatch.await();
 		Stopwatch stopwatch = Stopwatch.createStarted();
 		steadyLatch.await();
@@ -156,7 +156,10 @@ public void run() {
 				GlobalConstants.printOutputCountPeriod, TimeUnit.MILLISECONDS);
 	}
 
-	private void resetAll() {
+	/**
+	 * Releases all latches, and re inilizes the latches and counters.
+	 */
+	private void releaseAndInitilize() {
 		count = 0;
 		lastCount = 0;
 		skipLatch.countDown();
@@ -167,12 +170,6 @@ private void resetAll() {
 		steadyLatchUp = true;
 	}
 
-	/**
-	 * We need this method apart from {@link #resetAll()}, because the
-	 * {@link #resetAll()} method creates the latches immediately after
-	 * countDown(). This causes the threads which are waiting on the latches at
-	 * {@link #getFixedOutputTime()} will not be released properly.
-	 */
 	public void reset() {
 		steadyLatch.countDown();
 		skipLatch.countDown();

From 23d5920107f0ff81fd8e944c9ba7fd962329cc88 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 24 Jan 2015 19:01:59 +0800
Subject: [PATCH 559/881] New interface TailChannel has been added.

---
 .../impl/distributed/TailChannel.java         | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/TailChannel.java

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
new file mode 100644
index 00000000..687ade70
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -0,0 +1,61 @@
+package edu.mit.streamjit.impl.distributed;
+
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+
+/**
+ * This is a {@link BoundaryInputChannel} with counting facility.
+ * Implementations need to count the number of elements received and provide
+ * other services based on the count.
+ * 
+ * @author sumanan
+ * @since 24 Jan, 2015
+ */
+public interface TailChannel extends BoundaryInputChannel {
+
+	/**
+	 * @return Number of elements received after the last reset()
+	 */
+	public int count();
+
+	/**
+	 * Returns the time to receive fixed number of outputs. The fixed number can
+	 * be hard coded in side an implementation or passed as a constructor
+	 * argument.
+	 * <p>
+	 * The caller will be blocked until the fixed number of outputs are
+	 * received.
+	 * 
+	 * @return the time(ms) to receive fixed number of outputs.
+	 * 
+	 * @throws InterruptedException
+	 */
+	public long getFixedOutputTime() throws InterruptedException;
+
+	/**
+	 * Returns the time to receive fixed number of outputs. The fixed number can
+	 * be hard coded in side an implementation or passed as a constructor
+	 * argument. Waits at most <code>timeout</code> time to receive fixed number
+	 * of output. Returns -1 if timeout occurred.
+	 * 
+	 * <p>
+	 * The caller will be blocked until the fixed number of output is received
+	 * or timeout occurred, whatever happens early.
+	 * 
+	 * @param timeout
+	 *            Wait at most timeout time to receive fixed number of output.
+	 * 
+	 * @return the time(ms) to receive fixed number of outputs or -1 if timeout
+	 *         occurred.
+	 * 
+	 * @throws InterruptedException
+	 */
+	public long getFixedOutputTime(long timeout) throws InterruptedException;
+
+	/**
+	 * Resets all counters and other resources. Any thread blocked on either
+	 * {@link #getFixedOutputTime()} or {@link #getFixedOutputTime(long)} should
+	 * be released after this call.
+	 */
+	public void reset();
+
+}

From 3616c2353dbbd12ed1be0a187423d4faa7b53661 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 07:01:54 +0800
Subject: [PATCH 560/881] Refactored:TailChannels -> TailChannel1

TailChannel1 is static inner class of TailChannels.
---
 .../impl/distributed/StreamJitAppManager.java |   4 +-
 .../impl/distributed/TailChannels.java        | 414 +++++++++---------
 2 files changed, 220 insertions(+), 198 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 77140cd5..ff876999 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -87,7 +87,7 @@ public class StreamJitAppManager {
 	 * we need to pull the sink's output in to the {@link Controller} in order
 	 * to make {@link CompiledStream} .pull() to work.
 	 */
-	private TailChannels tailChannel;
+	private TailChannel tailChannel;
 
 	private Thread tailThread;
 
@@ -323,7 +323,7 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
-		tailChannel = new TailChannels(bufferMap.get(tailToken),
+		tailChannel = new TailChannels.TailChannel1(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
 						+ tailToken.toString(), 0, skipCount,
 				GlobalConstants.outputCount, app.name);
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 386af0d8..1c14c4f1 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -18,238 +18,260 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
-public class TailChannels extends BlockingInputChannel {
-
-	private final int skipCount;
-
-	private final int totalCount;
-
-	private int count;
-
-	private volatile CountDownLatch steadyLatch;
-
-	private volatile CountDownLatch skipLatch;
-
-	private PerformanceLogger pLogger = null;
-
-	private boolean skipLatchUp;
-
-	private boolean steadyLatchUp;
-
-	/**
-	 * Periodically prints no of outputs generated. See
-	 * {@link #printOutputCount()}.
-	 */
-	private ScheduledExecutorService scheduledExecutorService;
-
-	/**
-	 * The no of outputs generated at the end of last period. See
-	 * {@link #printOutputCount()}.
-	 */
-	private int lastCount;
-
-	/**
-	 * @param buffer
-	 * @param conProvider
-	 * @param conInfo
-	 * @param bufferTokenName
-	 * @param debugLevel
-	 * @param skipCount
-	 *            : Skips this amount of output before evaluating the running
-	 *            time. This is added to avoid the noise from init schedule and
-	 *            the drain data. ( i.e., In order to get real steady state
-	 *            execution time)
-	 * @param steadyCount
-	 *            : {@link #getFixedOutputTime()} calculates the time taken to
-	 *            get this amount of outputs ( after skipping skipCount number
-	 *            of outputs at the beginning).
-	 */
-	public TailChannels(Buffer buffer, ConnectionProvider conProvider,
-			ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
-			int skipCount, int steadyCount, String appName) {
-		super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
-		this.skipCount = skipCount;
-		this.totalCount = steadyCount + skipCount;
-		count = 0;
-		lastCount = 0;
-		steadyLatch = new CountDownLatch(1);
-		skipLatch = new CountDownLatch(1);
-		this.skipLatchUp = true;
-		this.steadyLatchUp = true;
-		if (GlobalConstants.tune == 0) {
-			pLogger = new PerformanceLogger(appName);
-			pLogger.start();
+public class TailChannels {
+
+	public static class TailChannel1 extends BlockingInputChannel implements
+			TailChannel {
+
+		private final int skipCount;
+
+		private final int totalCount;
+
+		private int count;
+
+		private volatile CountDownLatch steadyLatch;
+
+		private volatile CountDownLatch skipLatch;
+
+		private PerformanceLogger pLogger = null;
+
+		private boolean skipLatchUp;
+
+		private boolean steadyLatchUp;
+
+		/**
+		 * Periodically prints no of outputs generated. See
+		 * {@link #printOutputCount()}.
+		 */
+		private ScheduledExecutorService scheduledExecutorService;
+
+		/**
+		 * The no of outputs generated at the end of last period. See
+		 * {@link #printOutputCount()}.
+		 */
+		private int lastCount;
+
+		/**
+		 * @param buffer
+		 * @param conProvider
+		 * @param conInfo
+		 * @param bufferTokenName
+		 * @param debugLevel
+		 * @param skipCount
+		 *            : Skips this amount of output before evaluating the
+		 *            running time. This is added to avoid the noise from init
+		 *            schedule and the drain data. ( i.e., In order to get real
+		 *            steady state execution time)
+		 * @param steadyCount
+		 *            : {@link #getFixedOutputTime()} calculates the time taken
+		 *            to get this amount of outputs ( after skipping skipCount
+		 *            number of outputs at the beginning).
+		 */
+		public TailChannel1(Buffer buffer, ConnectionProvider conProvider,
+				ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
+				int skipCount, int steadyCount, String appName) {
+			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
+			this.skipCount = skipCount;
+			this.totalCount = steadyCount + skipCount;
+			count = 0;
+			lastCount = 0;
+			steadyLatch = new CountDownLatch(1);
+			skipLatch = new CountDownLatch(1);
+			this.skipLatchUp = true;
+			this.steadyLatchUp = true;
+			if (GlobalConstants.tune == 0) {
+				pLogger = new PerformanceLogger(appName);
+				pLogger.start();
+			}
+			printOutputCount();
 		}
-		printOutputCount();
-	}
 
-	@Override
-	public void receiveData() {
-		super.receiveData();
-		count++;
+		@Override
+		public void receiveData() {
+			super.receiveData();
+			count++;
 
-		if (skipLatchUp && count > skipCount) {
-			skipLatch.countDown();
-			skipLatchUp = false;
+			if (skipLatchUp && count > skipCount) {
+				skipLatch.countDown();
+				skipLatchUp = false;
+			}
+
+			if (steadyLatchUp && count > totalCount) {
+				steadyLatch.countDown();
+				steadyLatchUp = false;
+			}
 		}
 
-		if (steadyLatchUp && count > totalCount) {
-			steadyLatch.countDown();
-			steadyLatchUp = false;
+		@Override
+		public void stop(DrainType type) {
+			super.stop(type);
+			if (pLogger != null) {
+				releaseAndInitilize();
+				pLogger.stopLogging();
+			}
+			if (scheduledExecutorService != null)
+				scheduledExecutorService.shutdown();
 		}
-	}
 
-	@Override
-	public void stop(DrainType type) {
-		super.stop(type);
-		if (pLogger != null) {
+		/**
+		 * Skips skipCount amount of output at the beginning and then calculates
+		 * the time taken to get steadyCount amount of outputs. skipCount is
+		 * added to avoid the noise from init schedule and the drain data. (
+		 * i.e., In order to get real steady state execution time).
+		 * 
+		 * @return time in MILLISECONDS.
+		 * @throws InterruptedException
+		 */
+		public long getFixedOutputTime() throws InterruptedException {
 			releaseAndInitilize();
-			pLogger.stopLogging();
+			skipLatch.await();
+			Stopwatch stopwatch = Stopwatch.createStarted();
+			steadyLatch.await();
+			stopwatch.stop();
+			long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+			long normalizedTime = (GlobalConstants.outputCount * time)
+					/ (totalCount - skipCount);
+			return normalizedTime;
 		}
-		if (scheduledExecutorService != null)
-			scheduledExecutorService.shutdown();
-	}
-
-	/**
-	 * Skips skipCount amount of output at the beginning and then calculates the
-	 * time taken to get steadyCount amount of outputs. skipCount is added to
-	 * avoid the noise from init schedule and the drain data. ( i.e., In order
-	 * to get real steady state execution time).
-	 * 
-	 * @return time in MILLISECONDS.
-	 * @throws InterruptedException
-	 */
-	public long getFixedOutputTime() throws InterruptedException {
-		releaseAndInitilize();
-		skipLatch.await();
-		Stopwatch stopwatch = Stopwatch.createStarted();
-		steadyLatch.await();
-		stopwatch.stop();
-		long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-		long normalizedTime = (GlobalConstants.outputCount * time)
-				/ (totalCount - skipCount);
-		return normalizedTime;
-	}
 
-	/**
-	 * Periodically prints no of outputs generated.
-	 */
-	private void printOutputCount() {
-		if (GlobalConstants.printOutputCountPeriod < 1)
-			return;
+		/**
+		 * Periodically prints no of outputs generated.
+		 */
+		private void printOutputCount() {
+			if (GlobalConstants.printOutputCountPeriod < 1)
+				return;
 
-		lastCount = 0;
-		scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
-		scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+			lastCount = 0;
+			scheduledExecutorService = Executors
+					.newSingleThreadScheduledExecutor();
+			scheduledExecutorService.scheduleAtFixedRate(
+					new Runnable() {
+
+						@Override
+						public void run() {
+							int newOutputs = count - lastCount;
+							lastCount = count;
+							System.out.println(String
+									.format("Outputs: since started - %d, during last %d ms - %d",
+											count,
+											GlobalConstants.printOutputCountPeriod,
+											newOutputs));
+
+						}
+					}, GlobalConstants.printOutputCountPeriod,
+					GlobalConstants.printOutputCountPeriod,
+					TimeUnit.MILLISECONDS);
+		}
 
-			@Override
-			public void run() {
-				int newOutputs = count - lastCount;
-				lastCount = count;
-				System.out.println(String.format(
-						"Outputs: since started - %d, during last %d ms - %d",
-						count, GlobalConstants.printOutputCountPeriod,
-						newOutputs));
+		/**
+		 * Releases all latches, and re inilizes the latches and counters.
+		 */
+		private void releaseAndInitilize() {
+			count = 0;
+			lastCount = 0;
+			skipLatch.countDown();
+			skipLatch = new CountDownLatch(1);
+			skipLatchUp = true;
+			steadyLatch.countDown();
+			steadyLatch = new CountDownLatch(1);
+			steadyLatchUp = true;
+		}
 
-			}
-		}, GlobalConstants.printOutputCountPeriod,
-				GlobalConstants.printOutputCountPeriod, TimeUnit.MILLISECONDS);
-	}
+		public void reset() {
+			steadyLatch.countDown();
+			skipLatch.countDown();
+			count = 0;
+			lastCount = 0;
+		}
 
-	/**
-	 * Releases all latches, and re inilizes the latches and counters.
-	 */
-	private void releaseAndInitilize() {
-		count = 0;
-		lastCount = 0;
-		skipLatch.countDown();
-		skipLatch = new CountDownLatch(1);
-		skipLatchUp = true;
-		steadyLatch.countDown();
-		steadyLatch = new CountDownLatch(1);
-		steadyLatchUp = true;
-	}
+		private class PerformanceLogger extends Thread {
 
-	public void reset() {
-		steadyLatch.countDown();
-		skipLatch.countDown();
-		count = 0;
-		lastCount = 0;
-	}
+			private AtomicBoolean stopFlag;
 
-	private class PerformanceLogger extends Thread {
+			private final String appName;
 
-		private AtomicBoolean stopFlag;
+			private PerformanceLogger(String appName) {
+				super("PerformanceLogger");
+				stopFlag = new AtomicBoolean(false);
+				this.appName = appName;
+			}
 
-		private final String appName;
+			public void run() {
+				int i = 0;
+				FileWriter writer;
+				try {
+					writer = new FileWriter(String.format(
+							"%s%sFixedOutPut.txt", appName, File.separator));
+				} catch (IOException e1) {
+					e1.printStackTrace();
+					return;
+				}
 
-		private PerformanceLogger(String appName) {
-			super("PerformanceLogger");
-			stopFlag = new AtomicBoolean(false);
-			this.appName = appName;
-		}
+				writeInitialInfo(writer);
 
-		public void run() {
-			int i = 0;
-			FileWriter writer;
-			try {
-				writer = new FileWriter(String.format("%s%sFixedOutPut.txt",
-						appName, File.separator));
-			} catch (IOException e1) {
-				e1.printStackTrace();
-				return;
-			}
+				Long sum = 0l;
 
-			writeInitialInfo(writer);
+				while (++i < 10 && !stopFlag.get()) {
+					try {
+						Long time = getFixedOutputTime();
 
-			Long sum = 0l;
+						sum += time;
+						System.out.println("Execution time is " + time
+								+ " milli seconds");
 
-			while (++i < 10 && !stopFlag.get()) {
+						writer.write(time.toString());
+						writer.write('\n');
+						writer.flush();
+					} catch (InterruptedException | IOException e) {
+						e.printStackTrace();
+					}
+				}
 				try {
-					Long time = getFixedOutputTime();
+					writer.write("Average = " + sum / (i - 1));
+					writer.write('\n');
+					writer.flush();
+					writer.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+
+				System.out
+						.println("PerformanceLogger exits. App will run till "
+								+ "inputdata exhausted.");
+			}
 
-					sum += time;
-					System.out.println("Execution time is " + time
-							+ " milli seconds");
+			private void writeInitialInfo(FileWriter writer) {
+				System.out.println(String.format(
+						"PerformanceLogger starts to log the time to"
+								+ " produce %d number of outputs",
+						GlobalConstants.outputCount));
 
-					writer.write(time.toString());
+				try {
+					writer.write(String.format(
+							"GlobalConstants.outputCount = %d",
+							GlobalConstants.outputCount));
 					writer.write('\n');
 					writer.flush();
-				} catch (InterruptedException | IOException e) {
+				} catch (IOException e) {
 					e.printStackTrace();
 				}
 			}
-			try {
-				writer.write("Average = " + sum / (i - 1));
-				writer.write('\n');
-				writer.flush();
-				writer.close();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
 
-			System.out.println("PerformanceLogger exits. App will run till "
-					+ "inputdata exhausted.");
+			public void stopLogging() {
+				stopFlag.set(true);
+			}
 		}
 
-		private void writeInitialInfo(FileWriter writer) {
-			System.out.println(String.format(
-					"PerformanceLogger starts to log the time to"
-							+ " produce %d number of outputs",
-					GlobalConstants.outputCount));
-
-			try {
-				writer.write(String.format("GlobalConstants.outputCount = %d",
-						GlobalConstants.outputCount));
-				writer.write('\n');
-				writer.flush();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
+		@Override
+		public int count() {
+			return count;
 		}
 
-		public void stopLogging() {
-			stopFlag.set(true);
+		@Override
+		public long getFixedOutputTime(long timeout)
+				throws InterruptedException {
+			// TODO Auto-generated method stub
+			return 0;
 		}
 	}
 }
\ No newline at end of file

From 68c722f78a8d3a3d1b63e4b2f119f5a7944f6866 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 07:34:24 +0800
Subject: [PATCH 561/881] PerformanceLogger has been moved out

from TailChannel1 to TailChannels.
---
 .../impl/distributed/TailChannels.java        | 161 +++++++++---------
 1 file changed, 82 insertions(+), 79 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 1c14c4f1..6ee1e752 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -20,6 +20,85 @@
 
 public class TailChannels {
 
+	private static class PerformanceLogger extends Thread {
+
+		private AtomicBoolean stopFlag;
+
+		private final String appName;
+
+		private final TailChannel tailChannel;
+
+		private PerformanceLogger(TailChannel tailChannel, String appName) {
+			super("PerformanceLogger");
+			stopFlag = new AtomicBoolean(false);
+			this.appName = appName;
+			this.tailChannel = tailChannel;
+		}
+
+		public void run() {
+			int i = 0;
+			FileWriter writer;
+			try {
+				writer = new FileWriter(String.format("%s%sFixedOutPut.txt",
+						appName, File.separator));
+			} catch (IOException e1) {
+				e1.printStackTrace();
+				return;
+			}
+
+			writeInitialInfo(writer);
+
+			Long sum = 0l;
+
+			while (++i < 10 && !stopFlag.get()) {
+				try {
+					Long time = tailChannel.getFixedOutputTime();
+
+					sum += time;
+					System.out.println("Execution time is " + time
+							+ " milli seconds");
+
+					writer.write(time.toString());
+					writer.write('\n');
+					writer.flush();
+				} catch (InterruptedException | IOException e) {
+					e.printStackTrace();
+				}
+			}
+			try {
+				writer.write("Average = " + sum / (i - 1));
+				writer.write('\n');
+				writer.flush();
+				writer.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+
+			System.out.println("PerformanceLogger exits. App will run till "
+					+ "inputdata exhausted.");
+		}
+
+		private void writeInitialInfo(FileWriter writer) {
+			System.out.println(String.format(
+					"PerformanceLogger starts to log the time to"
+							+ " produce %d number of outputs",
+					GlobalConstants.outputCount));
+
+			try {
+				writer.write(String.format("GlobalConstants.outputCount = %d",
+						GlobalConstants.outputCount));
+				writer.write('\n');
+				writer.flush();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+
+		public void stopLogging() {
+			stopFlag.set(true);
+		}
+	}
+
 	public static class TailChannel1 extends BlockingInputChannel implements
 			TailChannel {
 
@@ -80,7 +159,9 @@ public TailChannel1(Buffer buffer, ConnectionProvider conProvider,
 			this.skipLatchUp = true;
 			this.steadyLatchUp = true;
 			if (GlobalConstants.tune == 0) {
-				pLogger = new PerformanceLogger(appName);
+				// TODO: Leaks this object from the constructor. May cause
+				// subtle bugs. Re-factor it.
+				pLogger = new PerformanceLogger(this, appName);
 				pLogger.start();
 			}
 			printOutputCount();
@@ -184,84 +265,6 @@ public void reset() {
 			lastCount = 0;
 		}
 
-		private class PerformanceLogger extends Thread {
-
-			private AtomicBoolean stopFlag;
-
-			private final String appName;
-
-			private PerformanceLogger(String appName) {
-				super("PerformanceLogger");
-				stopFlag = new AtomicBoolean(false);
-				this.appName = appName;
-			}
-
-			public void run() {
-				int i = 0;
-				FileWriter writer;
-				try {
-					writer = new FileWriter(String.format(
-							"%s%sFixedOutPut.txt", appName, File.separator));
-				} catch (IOException e1) {
-					e1.printStackTrace();
-					return;
-				}
-
-				writeInitialInfo(writer);
-
-				Long sum = 0l;
-
-				while (++i < 10 && !stopFlag.get()) {
-					try {
-						Long time = getFixedOutputTime();
-
-						sum += time;
-						System.out.println("Execution time is " + time
-								+ " milli seconds");
-
-						writer.write(time.toString());
-						writer.write('\n');
-						writer.flush();
-					} catch (InterruptedException | IOException e) {
-						e.printStackTrace();
-					}
-				}
-				try {
-					writer.write("Average = " + sum / (i - 1));
-					writer.write('\n');
-					writer.flush();
-					writer.close();
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-
-				System.out
-						.println("PerformanceLogger exits. App will run till "
-								+ "inputdata exhausted.");
-			}
-
-			private void writeInitialInfo(FileWriter writer) {
-				System.out.println(String.format(
-						"PerformanceLogger starts to log the time to"
-								+ " produce %d number of outputs",
-						GlobalConstants.outputCount));
-
-				try {
-					writer.write(String.format(
-							"GlobalConstants.outputCount = %d",
-							GlobalConstants.outputCount));
-					writer.write('\n');
-					writer.flush();
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			}
-
-			public void stopLogging() {
-				stopFlag.set(true);
-			}
-		}
-
 		@Override
 		public int count() {
 			return count;

From 0c50b2be77af1164fe68ca8882fcd95e572bb679 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 08:25:31 +0800
Subject: [PATCH 562/881] OutputCountPrinter added.

---
 .../impl/distributed/TailChannels.java        | 107 ++++++++++--------
 1 file changed, 60 insertions(+), 47 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 6ee1e752..b89d34c5 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -99,6 +99,62 @@ public void stopLogging() {
 		}
 	}
 
+	private static class OutputCountPrinter {
+
+		private final TailChannel tailChannel;
+
+		/**
+		 * The no of outputs generated at the end of last period. See
+		 * {@link #printOutputCount()}.
+		 */
+		private int lastCount;
+
+		/**
+		 * Periodically prints no of outputs generated. See
+		 * {@link #printOutputCount()}.
+		 */
+		private ScheduledExecutorService scheduledExecutorService;
+
+		OutputCountPrinter(TailChannel tailChannel) {
+			this.tailChannel = tailChannel;
+			printOutputCount();
+		}
+
+		/**
+		 * Periodically prints no of outputs generated.
+		 */
+		private void printOutputCount() {
+			if (GlobalConstants.printOutputCountPeriod < 1)
+				return;
+
+			lastCount = 0;
+			scheduledExecutorService = Executors
+					.newSingleThreadScheduledExecutor();
+			scheduledExecutorService.scheduleAtFixedRate(
+					new Runnable() {
+
+						@Override
+						public void run() {
+							int newOutputs = tailChannel.count() - lastCount;
+							lastCount = tailChannel.count();
+							System.out.println(String
+									.format("Outputs: since started - %d, during last %d ms - %d",
+											tailChannel.count(),
+											GlobalConstants.printOutputCountPeriod,
+											newOutputs));
+
+						}
+					}, GlobalConstants.printOutputCountPeriod,
+					GlobalConstants.printOutputCountPeriod,
+					TimeUnit.MILLISECONDS);
+		}
+
+		private void stop() {
+			if (scheduledExecutorService != null)
+				scheduledExecutorService.shutdown();
+		}
+	}
+
 	public static class TailChannel1 extends BlockingInputChannel implements
 			TailChannel {
 
@@ -114,22 +170,12 @@ public static class TailChannel1 extends BlockingInputChannel implements
 
 		private PerformanceLogger pLogger = null;
 
+		private OutputCountPrinter outputCountPrinter;
+
 		private boolean skipLatchUp;
 
 		private boolean steadyLatchUp;
 
-		/**
-		 * Periodically prints no of outputs generated. See
-		 * {@link #printOutputCount()}.
-		 */
-		private ScheduledExecutorService scheduledExecutorService;
-
-		/**
-		 * The no of outputs generated at the end of last period. See
-		 * {@link #printOutputCount()}.
-		 */
-		private int lastCount;
-
 		/**
 		 * @param buffer
 		 * @param conProvider
@@ -153,7 +199,6 @@ public TailChannel1(Buffer buffer, ConnectionProvider conProvider,
 			this.skipCount = skipCount;
 			this.totalCount = steadyCount + skipCount;
 			count = 0;
-			lastCount = 0;
 			steadyLatch = new CountDownLatch(1);
 			skipLatch = new CountDownLatch(1);
 			this.skipLatchUp = true;
@@ -164,7 +209,7 @@ public TailChannel1(Buffer buffer, ConnectionProvider conProvider,
 				pLogger = new PerformanceLogger(this, appName);
 				pLogger.start();
 			}
-			printOutputCount();
+			outputCountPrinter = new OutputCountPrinter(this);
 		}
 
 		@Override
@@ -190,8 +235,7 @@ public void stop(DrainType type) {
 				releaseAndInitilize();
 				pLogger.stopLogging();
 			}
-			if (scheduledExecutorService != null)
-				scheduledExecutorService.shutdown();
+			outputCountPrinter.stop();
 		}
 
 		/**
@@ -215,41 +259,11 @@ public long getFixedOutputTime() throws InterruptedException {
 			return normalizedTime;
 		}
 
-		/**
-		 * Periodically prints no of outputs generated.
-		 */
-		private void printOutputCount() {
-			if (GlobalConstants.printOutputCountPeriod < 1)
-				return;
-
-			lastCount = 0;
-			scheduledExecutorService = Executors
-					.newSingleThreadScheduledExecutor();
-			scheduledExecutorService.scheduleAtFixedRate(
-					new Runnable() {
-
-						@Override
-						public void run() {
-							int newOutputs = count - lastCount;
-							lastCount = count;
-							System.out.println(String
-									.format("Outputs: since started - %d, during last %d ms - %d",
-											count,
-											GlobalConstants.printOutputCountPeriod,
-											newOutputs));
-
-						}
-					}, GlobalConstants.printOutputCountPeriod,
-					GlobalConstants.printOutputCountPeriod,
-					TimeUnit.MILLISECONDS);
-		}
-
 		/**
 		 * Releases all latches, and re inilizes the latches and counters.
 		 */
 		private void releaseAndInitilize() {
 			count = 0;
-			lastCount = 0;
 			skipLatch.countDown();
 			skipLatch = new CountDownLatch(1);
 			skipLatchUp = true;
@@ -262,7 +276,6 @@ public void reset() {
 			steadyLatch.countDown();
 			skipLatch.countDown();
 			count = 0;
-			lastCount = 0;
 		}
 
 		@Override

From c449757cef6d28c0f183ce157f734a3464da1ba2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 08:28:26 +0800
Subject: [PATCH 563/881] OutputCountPrinter calls tailChannel.count() ...

only once and store the value in a local variable.
---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index b89d34c5..cc3e7aaf 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -135,11 +135,12 @@ private void printOutputCount() {
 
 						@Override
 						public void run() {
-							int newOutputs = tailChannel.count() - lastCount;
-							lastCount = tailChannel.count();
+							int currentCount = tailChannel.count();
+							int newOutputs = currentCount - lastCount;
+							lastCount = currentCount;
 							System.out.println(String
 									.format("Outputs: since started - %d, during last %d ms - %d",
-											tailChannel.count(),
+											currentCount,
 											GlobalConstants.printOutputCountPeriod,
 											newOutputs));
 

From c4c88b0f9295526b2ad09033a799eaa54143975f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 08:33:00 +0800
Subject: [PATCH 564/881] Renamed:TailChannel1->BlockingTailChannel1

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java   | 2 +-
 src/edu/mit/streamjit/impl/distributed/TailChannels.java      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index ff876999..9f00a97f 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -323,7 +323,7 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
-		tailChannel = new TailChannels.TailChannel1(bufferMap.get(tailToken),
+		tailChannel = new TailChannels.BlockingTailChannel1(bufferMap.get(tailToken),
 				controller.getConProvider(), tailconInfo, "tailChannel - "
 						+ tailToken.toString(), 0, skipCount,
 				GlobalConstants.outputCount, app.name);
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index cc3e7aaf..23e43d39 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -156,7 +156,7 @@ private void stop() {
 		}
 	}
 
-	public static class TailChannel1 extends BlockingInputChannel implements
+	public static class BlockingTailChannel1 extends BlockingInputChannel implements
 			TailChannel {
 
 		private final int skipCount;
@@ -193,7 +193,7 @@ public static class TailChannel1 extends BlockingInputChannel implements
 		 *            to get this amount of outputs ( after skipping skipCount
 		 *            number of outputs at the beginning).
 		 */
-		public TailChannel1(Buffer buffer, ConnectionProvider conProvider,
+		public BlockingTailChannel1(Buffer buffer, ConnectionProvider conProvider,
 				ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
 				int skipCount, int steadyCount, String appName) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);

From 915162b361113639e86fa97461030c57a92f4ba9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 08:42:18 +0800
Subject: [PATCH 565/881] Creates OutputCountPrinter iff it is needed.

---
 .../impl/distributed/TailChannels.java        | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 23e43d39..7a426cc7 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -156,8 +156,8 @@ private void stop() {
 		}
 	}
 
-	public static class BlockingTailChannel1 extends BlockingInputChannel implements
-			TailChannel {
+	public static class BlockingTailChannel1 extends BlockingInputChannel
+			implements TailChannel {
 
 		private final int skipCount;
 
@@ -171,7 +171,7 @@ public static class BlockingTailChannel1 extends BlockingInputChannel implements
 
 		private PerformanceLogger pLogger = null;
 
-		private OutputCountPrinter outputCountPrinter;
+		private OutputCountPrinter outputCountPrinter = null;
 
 		private boolean skipLatchUp;
 
@@ -193,9 +193,10 @@ public static class BlockingTailChannel1 extends BlockingInputChannel implements
 		 *            to get this amount of outputs ( after skipping skipCount
 		 *            number of outputs at the beginning).
 		 */
-		public BlockingTailChannel1(Buffer buffer, ConnectionProvider conProvider,
-				ConnectionInfo conInfo, String bufferTokenName, int debugLevel,
-				int skipCount, int steadyCount, String appName) {
+		public BlockingTailChannel1(Buffer buffer,
+				ConnectionProvider conProvider, ConnectionInfo conInfo,
+				String bufferTokenName, int debugLevel, int skipCount,
+				int steadyCount, String appName) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 			this.skipCount = skipCount;
 			this.totalCount = steadyCount + skipCount;
@@ -210,7 +211,8 @@ public BlockingTailChannel1(Buffer buffer, ConnectionProvider conProvider,
 				pLogger = new PerformanceLogger(this, appName);
 				pLogger.start();
 			}
-			outputCountPrinter = new OutputCountPrinter(this);
+			if (GlobalConstants.printOutputCountPeriod > 0)
+				outputCountPrinter = new OutputCountPrinter(this);
 		}
 
 		@Override
@@ -236,7 +238,8 @@ public void stop(DrainType type) {
 				releaseAndInitilize();
 				pLogger.stopLogging();
 			}
-			outputCountPrinter.stop();
+			if (outputCountPrinter != null)
+				outputCountPrinter.stop();
 		}
 
 		/**

From 525160f0cc1b9549e77b9af261948c3e3d75fb0b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 08:54:35 +0800
Subject: [PATCH 566/881] Changes in comments after the refactoring.

---
 .../streamjit/impl/distributed/TailChannels.java | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 7a426cc7..71d7710d 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -99,20 +99,19 @@ public void stopLogging() {
 		}
 	}
 
+	/**
+	 * Periodically prints the number of outputs received by a
+	 * {@link TailChannel}.
+	 */
 	private static class OutputCountPrinter {
 
 		private final TailChannel tailChannel;
 
 		/**
-		 * The no of outputs generated at the end of last period. See
-		 * {@link #printOutputCount()}.
+		 * The no of outputs received at the end of last period.
 		 */
 		private int lastCount;
 
-		/**
-		 * Periodically prints no of outputs generated. See
-		 * {@link #printOutputCount()}.
-		 */
 		private ScheduledExecutorService scheduledExecutorService;
 
 		OutputCountPrinter(TailChannel tailChannel) {
@@ -120,9 +119,6 @@ private static class OutputCountPrinter {
 			printOutputCount();
 		}
 
-		/**
-		 * Periodically prints no of outputs generated.
-		 */
 		private void printOutputCount() {
 			if (GlobalConstants.printOutputCountPeriod < 1)
 				return;
@@ -264,7 +260,7 @@ public long getFixedOutputTime() throws InterruptedException {
 		}
 
 		/**
-		 * Releases all latches, and re inilizes the latches and counters.
+		 * Releases all latches, and re-initializes the latches and counters.
 		 */
 		private void releaseAndInitilize() {
 			count = 0;

From 7b1a100aa00ee2f941d1e8ae06c00a192c4f3a7c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 09:38:18 +0800
Subject: [PATCH 567/881] BlockingTailChannel.normalizedTime() added.

---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 71d7710d..5b249d10 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -254,9 +254,7 @@ public long getFixedOutputTime() throws InterruptedException {
 			steadyLatch.await();
 			stopwatch.stop();
 			long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-			long normalizedTime = (GlobalConstants.outputCount * time)
-					/ (totalCount - skipCount);
-			return normalizedTime;
+			return normalizedTime(time);
 		}
 
 		/**
@@ -289,5 +287,10 @@ public long getFixedOutputTime(long timeout)
 			// TODO Auto-generated method stub
 			return 0;
 		}
+
+		private long normalizedTime(long time) {
+			return (GlobalConstants.outputCount * time)
+					/ (totalCount - skipCount);
+		}
 	}
 }
\ No newline at end of file

From 6b9fe254f32d5f11326a475e8876da5c2c1949ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 10:25:38 +0800
Subject: [PATCH 568/881] unnormalizedTime() added.

---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 5b249d10..de43bb3e 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -292,5 +292,14 @@ private long normalizedTime(long time) {
 			return (GlobalConstants.outputCount * time)
 					/ (totalCount - skipCount);
 		}
+
+		/**
+		 * Opposite to the {@link #normalizedTime(long)}'s equation.
+		 * <code>time=unnormalizedTime(normalizedTime(time))</code>
+		 */
+		private long unnormalizedTime(long time) {
+			return (time * (totalCount - skipCount))
+					/ GlobalConstants.outputCount;
+		}
 	}
 }
\ No newline at end of file

From 233a84670025eb98c77eb9124482c1f28fa30752 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 10:27:00 +0800
Subject: [PATCH 569/881] getFixedOutputTime(long timeOut) has been moved up

---
 .../streamjit/impl/distributed/TailChannels.java    | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index de43bb3e..a80d78f6 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -257,6 +257,12 @@ public long getFixedOutputTime() throws InterruptedException {
 			return normalizedTime(time);
 		}
 
+		@Override
+		public long getFixedOutputTime(long timeout)
+				throws InterruptedException {
+			return 0;
+		}
+
 		/**
 		 * Releases all latches, and re-initializes the latches and counters.
 		 */
@@ -281,13 +287,6 @@ public int count() {
 			return count;
 		}
 
-		@Override
-		public long getFixedOutputTime(long timeout)
-				throws InterruptedException {
-			// TODO Auto-generated method stub
-			return 0;
-		}
-
 		private long normalizedTime(long time) {
 			return (GlobalConstants.outputCount * time)
 					/ (totalCount - skipCount);

From 27cab8a43976b0ab6a538590fa42ddc0b7721831 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 10:35:21 +0800
Subject: [PATCH 570/881] getFixedOutputTime(long timeout) implemented.

---
 .../streamjit/impl/distributed/TailChannels.java | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index a80d78f6..ce4ad863 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -260,7 +260,21 @@ public long getFixedOutputTime() throws InterruptedException {
 		@Override
 		public long getFixedOutputTime(long timeout)
 				throws InterruptedException {
-			return 0;
+			timeout = unnormalizedTime(timeout);
+			releaseAndInitilize();
+			skipLatch.await();
+			Stopwatch stopwatch = Stopwatch.createStarted();
+			while (steadyLatch.getCount() > 0
+					&& stopwatch.elapsed(TimeUnit.MILLISECONDS) < timeout) {
+				Thread.sleep(100);
+			}
+
+			if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > timeout)
+				return -1;
+
+			stopwatch.stop();
+			long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+			return normalizedTime(time);
 		}
 
 		/**

From 524693ecce855cf3240927782868dcebc4220fcc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 10:48:09 +0800
Subject: [PATCH 571/881] If timeOut< 1, getFixedOutputTime(timeOut)

If timeout < 1, then the behavior getFixedOutputTime(timeOut) method is
equivalent to calling getFixedOutputTime().
---
 src/edu/mit/streamjit/impl/distributed/TailChannel.java  | 5 +++++
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 3 +++
 2 files changed, 8 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannel.java b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
index 687ade70..d9082399 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannel.java
@@ -38,6 +38,11 @@ public interface TailChannel extends BoundaryInputChannel {
 	 * of output. Returns -1 if timeout occurred.
 	 * 
 	 * <p>
+	 * If timeout < 1, then the behavior this method is equivalent to calling
+	 * {@link #getFixedOutputTime()}.
+	 * </p>
+	 * 
+	 * <p>
 	 * The caller will be blocked until the fixed number of output is received
 	 * or timeout occurred, whatever happens early.
 	 * 
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index ce4ad863..2da6afe2 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -260,6 +260,9 @@ public long getFixedOutputTime() throws InterruptedException {
 		@Override
 		public long getFixedOutputTime(long timeout)
 				throws InterruptedException {
+			if (timeout < 1)
+				return getFixedOutputTime();
+
 			timeout = unnormalizedTime(timeout);
 			releaseAndInitilize();
 			skipLatch.await();

From 570d0c5ec496f99f75a3b86755ad73fbda3a9a3d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 13:00:33 +0800
Subject: [PATCH 572/881] Uses getFixedOutputTime(long timeout)

OnlineTuner and StreamJitAppManager uses
TailChannel.getFixedOutputTime(long timeout) instead of
getFixedOutputTime().
---
 .../impl/distributed/StreamJitAppManager.java       |  4 ++--
 .../impl/distributed/runtimer/OnlineTuner.java      | 13 ++++++++-----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 9f00a97f..578268ec 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -210,8 +210,8 @@ public SNTimeInfoProcessor timeInfoProcessor() {
 		return timeInfoProcessor;
 	}
 
-	public long getFixedOutputTime() throws InterruptedException {
-		long time = tailChannel.getFixedOutputTime();
+	public long getFixedOutputTime(long timeout) throws InterruptedException {
+		long time = tailChannel.getFixedOutputTime(timeout);
 		if (apStsPro.error) {
 			return -1l;
 		}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2083a90d..ce117f81 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -93,7 +93,7 @@ private void tune() {
 					ConfigurationUtils.saveConfg(cfgJson,
 							new Integer(round).toString(), app.name);
 
-				ret = reconfigure(config);
+				ret = reconfigure(config, 0);
 				if (ret.first) {
 					prognosticator.time(ret.second);
 					tuner.writeLine(new Double(ret.second).toString());
@@ -173,12 +173,15 @@ private void verifyTuningTimes() {
 	}
 
 	/**
+	 * TODO: Split this method into two methods, 1.reconfigure(),
+	 * 2.getFixedOutputTime().
+	 * 
 	 * @param cfgJson
 	 * @param round
 	 * @return if ret.first == false, then no more tuning. ret.second = running
 	 *         time in milliseconds.
 	 */
-	private Pair<Boolean, Long> reconfigure(Configuration config) {
+	private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		long time;
 
 		if (manager.getStatus() == AppStatus.STOPPED)
@@ -200,7 +203,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config) {
 				// TODO: need to check the manager's status before passing the
 				// time. Exceptions, final drain, etc may causes app to stop
 				// executing.
-				time = manager.getFixedOutputTime();
+				time = manager.getFixedOutputTime(timeout);
 				logger.logRunTime(time);
 			} else {
 				time = -1l;
@@ -259,7 +262,7 @@ private void handleTermination() throws IOException {
 		if (needTermination) {
 			terminate();
 		} else {
-			Pair<Boolean, Long> ret = reconfigure(finalcfg);
+			Pair<Boolean, Long> ret = reconfigure(finalcfg, 0);
 			if (ret.first && ret.second > 0)
 				System.out
 						.println("Application is running forever with the final configuration.");
@@ -298,7 +301,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			if (cfg != null) {
 				Pair<Boolean, Long> ret;
 				for (int i = 0; i < count; i++) {
-					ret = reconfigure(cfg);
+					ret = reconfigure(cfg, 0);
 					if (ret.first) {
 						prognosticator.time(ret.second);
 						writer.write(ret.second.toString());

From 698abfcbc8f60e0ab9552ca7f9799f25a7ea61c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 25 Jan 2015 13:18:52 +0800
Subject: [PATCH 573/881] tune() uses currentBestTime.

OnlineTuner.tune() keeps track of currentBestTime and use that value to discard
the bad configurations early.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index ce117f81..6e4b713e 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -62,6 +62,9 @@ else if (GlobalConstants.tune == 2)
 
 	private void tune() {
 		int round = 0;
+		// Keeps track of the current best time. Uses this to discard bad cfgs
+		// early.
+		long currentBestTime = Long.MAX_VALUE;
 		try {
 			startTuner();
 			Pair<Boolean, Long> ret;
@@ -93,8 +96,11 @@ private void tune() {
 					ConfigurationUtils.saveConfg(cfgJson,
 							new Integer(round).toString(), app.name);
 
-				ret = reconfigure(config, 0);
+				ret = reconfigure(config, 2 * currentBestTime);
 				if (ret.first) {
+					long time = ret.second;
+					currentBestTime = (time > 1 && currentBestTime > time)
+							? time : currentBestTime;
 					prognosticator.time(ret.second);
 					tuner.writeLine(new Double(ret.second).toString());
 				} else {

From cc70462b88d20d545ea7824c3269d28f1fc4f3ea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 10:36:50 +0800
Subject: [PATCH 574/881] AbstractBlockingTailChannel added.

---
 .../mit/streamjit/impl/distributed/TailChannels.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 2da6afe2..1b7ecfab 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -152,6 +152,18 @@ private void stop() {
 		}
 	}
 
+	private static abstract class AbstractBlockingTailChannel
+			extends
+				BlockingInputChannel implements TailChannel {
+
+		public AbstractBlockingTailChannel(Buffer buffer,
+				ConnectionProvider conProvider, ConnectionInfo conInfo,
+				String bufferTokenName, int debugLevel) {
+			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
+		}
+
+	}
+
 	public static class BlockingTailChannel1 extends BlockingInputChannel
 			implements TailChannel {
 

From 0d58a5037fb5f58dc0812744946cb247a1bd3886 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 10:50:00 +0800
Subject: [PATCH 575/881] Refactored:AbsBlockingTailChl&BlockingTailChnl1

Some methods and some instance variables from BlockingTailChannel1 have been
moved to AbstractBlockingTailChannel.
---
 .../impl/distributed/TailChannels.java        | 128 ++++++++++--------
 1 file changed, 74 insertions(+), 54 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 1b7ecfab..0fe41e2c 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -156,31 +156,91 @@ private static abstract class AbstractBlockingTailChannel
 			extends
 				BlockingInputChannel implements TailChannel {
 
+		protected final int skipCount;
+
+		protected final int totalCount;
+
+		protected int count;
+
+		private PerformanceLogger pLogger = null;
+
+		private OutputCountPrinter outputCountPrinter = null;
+
+		protected abstract void releaseAndInitilize();
+
+		/**
+		 * @param buffer
+		 * @param conProvider
+		 * @param conInfo
+		 * @param bufferTokenName
+		 * @param debugLevel
+		 * @param skipCount
+		 *            : Skips this amount of output before evaluating the
+		 *            running time. This is added to avoid the noise from init
+		 *            schedule and the drain data. ( i.e., In order to get real
+		 *            steady state execution time)
+		 * @param steadyCount
+		 *            : {@link #getFixedOutputTime()} calculates the time taken
+		 *            to get this amount of outputs ( after skipping skipCount
+		 *            number of outputs at the beginning).
+		 */
 		public AbstractBlockingTailChannel(Buffer buffer,
 				ConnectionProvider conProvider, ConnectionInfo conInfo,
-				String bufferTokenName, int debugLevel) {
+				String bufferTokenName, int debugLevel, int skipCount,
+				int steadyCount, String appName) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
+			this.skipCount = skipCount;
+			this.totalCount = steadyCount + skipCount;
+			count = 0;
+			if (GlobalConstants.tune == 0) {
+				// TODO: Leaks this object from the constructor. May cause
+				// subtle bugs. Re-factor it.
+				pLogger = new PerformanceLogger(this, appName);
+				pLogger.start();
+			}
+			if (GlobalConstants.printOutputCountPeriod > 0)
+				outputCountPrinter = new OutputCountPrinter(this);
 		}
 
-	}
+		@Override
+		public void stop(DrainType type) {
+			super.stop(type);
+			if (pLogger != null) {
+				releaseAndInitilize();
+				pLogger.stopLogging();
+			}
+			if (outputCountPrinter != null)
+				outputCountPrinter.stop();
+		}
 
-	public static class BlockingTailChannel1 extends BlockingInputChannel
-			implements TailChannel {
+		@Override
+		public int count() {
+			return count;
+		}
 
-		private final int skipCount;
+		protected long normalizedTime(long time) {
+			return (GlobalConstants.outputCount * time)
+					/ (totalCount - skipCount);
+		}
 
-		private final int totalCount;
+		/**
+		 * Opposite to the {@link #normalizedTime(long)}'s equation.
+		 * <code>time=unnormalizedTime(normalizedTime(time))</code>
+		 */
+		protected long unnormalizedTime(long time) {
+			return (time * (totalCount - skipCount))
+					/ GlobalConstants.outputCount;
+		}
+	}
 
-		private int count;
+	public static final class BlockingTailChannel1
+			extends
+				AbstractBlockingTailChannel {
 
 		private volatile CountDownLatch steadyLatch;
 
 		private volatile CountDownLatch skipLatch;
 
-		private PerformanceLogger pLogger = null;
-
-		private OutputCountPrinter outputCountPrinter = null;
-
 		private boolean skipLatchUp;
 
 		private boolean steadyLatchUp;
@@ -205,22 +265,12 @@ public BlockingTailChannel1(Buffer buffer,
 				ConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel, int skipCount,
 				int steadyCount, String appName) {
-			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
-			this.skipCount = skipCount;
-			this.totalCount = steadyCount + skipCount;
-			count = 0;
+			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel,
+					skipCount, steadyCount, appName);
 			steadyLatch = new CountDownLatch(1);
 			skipLatch = new CountDownLatch(1);
 			this.skipLatchUp = true;
 			this.steadyLatchUp = true;
-			if (GlobalConstants.tune == 0) {
-				// TODO: Leaks this object from the constructor. May cause
-				// subtle bugs. Re-factor it.
-				pLogger = new PerformanceLogger(this, appName);
-				pLogger.start();
-			}
-			if (GlobalConstants.printOutputCountPeriod > 0)
-				outputCountPrinter = new OutputCountPrinter(this);
 		}
 
 		@Override
@@ -239,17 +289,6 @@ public void receiveData() {
 			}
 		}
 
-		@Override
-		public void stop(DrainType type) {
-			super.stop(type);
-			if (pLogger != null) {
-				releaseAndInitilize();
-				pLogger.stopLogging();
-			}
-			if (outputCountPrinter != null)
-				outputCountPrinter.stop();
-		}
-
 		/**
 		 * Skips skipCount amount of output at the beginning and then calculates
 		 * the time taken to get steadyCount amount of outputs. skipCount is
@@ -295,7 +334,7 @@ public long getFixedOutputTime(long timeout)
 		/**
 		 * Releases all latches, and re-initializes the latches and counters.
 		 */
-		private void releaseAndInitilize() {
+		protected void releaseAndInitilize() {
 			count = 0;
 			skipLatch.countDown();
 			skipLatch = new CountDownLatch(1);
@@ -310,24 +349,5 @@ public void reset() {
 			skipLatch.countDown();
 			count = 0;
 		}
-
-		@Override
-		public int count() {
-			return count;
-		}
-
-		private long normalizedTime(long time) {
-			return (GlobalConstants.outputCount * time)
-					/ (totalCount - skipCount);
-		}
-
-		/**
-		 * Opposite to the {@link #normalizedTime(long)}'s equation.
-		 * <code>time=unnormalizedTime(normalizedTime(time))</code>
-		 */
-		private long unnormalizedTime(long time) {
-			return (time * (totalCount - skipCount))
-					/ GlobalConstants.outputCount;
-		}
 	}
 }
\ No newline at end of file

From e06f839b7f233e54f250846db771fb9e04b11cb6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 11:05:22 +0800
Subject: [PATCH 576/881] Comments added to BlockingTailChannel1's constr.

---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 0fe41e2c..436ea22a 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -251,6 +251,8 @@ public static final class BlockingTailChannel1
 		 * @param conInfo
 		 * @param bufferTokenName
 		 * @param debugLevel
+		 *            For all above 5 parameters, see
+		 *            {@link BlockingInputChannel#BlockingInputChannel(Buffer, ConnectionProvider, ConnectionInfo, String, int)}
 		 * @param skipCount
 		 *            : Skips this amount of output before evaluating the
 		 *            running time. This is added to avoid the noise from init

From a50a5ce0cd8badcd4951fa79761bae53d2020354 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 11:44:57 +0800
Subject: [PATCH 577/881] BlockingTailChannel2 added.

This uses StopWatch instead of CountDownLatch to getFixedOutputTime().
---
 .../impl/distributed/TailChannels.java        | 114 ++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 436ea22a..b4158bf4 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -352,4 +352,118 @@ public void reset() {
 			count = 0;
 		}
 	}
+
+	public static final class BlockingTailChannel2
+			extends
+				AbstractBlockingTailChannel {
+
+		private volatile CountDownLatch skipLatch;
+
+		private boolean skipLatchUp;
+
+		private Stopwatch stopWatch;
+
+		/**
+		 * @param buffer
+		 * @param conProvider
+		 * @param conInfo
+		 * @param bufferTokenName
+		 * @param debugLevel
+		 *            For all above 5 parameters, see
+		 *            {@link BlockingInputChannel#BlockingInputChannel(Buffer, ConnectionProvider, ConnectionInfo, String, int)}
+		 * @param skipCount
+		 *            : Skips this amount of output before evaluating the
+		 *            running time. This is added to avoid the noise from init
+		 *            schedule and the drain data. ( i.e., In order to get real
+		 *            steady state execution time)
+		 * @param steadyCount
+		 *            : {@link #getFixedOutputTime()} calculates the time taken
+		 *            to get this amount of outputs ( after skipping skipCount
+		 *            number of outputs at the beginning).
+		 */
+		public BlockingTailChannel2(Buffer buffer,
+				ConnectionProvider conProvider, ConnectionInfo conInfo,
+				String bufferTokenName, int debugLevel, int skipCount,
+				int steadyCount, String appName) {
+			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel,
+					skipCount, steadyCount, appName);
+			stopWatch = Stopwatch.createUnstarted();
+			skipLatch = new CountDownLatch(1);
+			this.skipLatchUp = true;
+		}
+
+		@Override
+		public void receiveData() {
+			super.receiveData();
+			count++;
+
+			if (skipLatchUp && count > skipCount) {
+				skipLatch.countDown();
+				skipLatchUp = false;
+			}
+
+			if (stopWatch.isRunning() && count > totalCount) {
+				stopWatch.stop();
+			}
+		}
+
+		/**
+		 * Skips skipCount amount of output at the beginning and then calculates
+		 * the time taken to get steadyCount amount of outputs. skipCount is
+		 * added to avoid the noise from init schedule and the drain data. (
+		 * i.e., In order to get real steady state execution time).
+		 * 
+		 * @return time in MILLISECONDS.
+		 * @throws InterruptedException
+		 */
+		public long getFixedOutputTime() throws InterruptedException {
+			releaseAndInitilize();
+			skipLatch.await();
+			stopWatch.start();
+			while (stopWatch.isRunning())
+				Thread.sleep(250);
+			long time = stopWatch.elapsed(TimeUnit.MILLISECONDS);
+			return normalizedTime(time);
+		}
+
+		@Override
+		public long getFixedOutputTime(long timeout)
+				throws InterruptedException {
+			if (timeout < 1)
+				return getFixedOutputTime();
+
+			timeout = unnormalizedTime(timeout);
+			releaseAndInitilize();
+			skipLatch.await();
+			stopWatch.start();
+			while (stopWatch.isRunning()
+					&& stopWatch.elapsed(TimeUnit.MILLISECONDS) < timeout) {
+				Thread.sleep(250);
+			}
+
+			long time = stopWatch.elapsed(TimeUnit.MILLISECONDS);
+
+			if (time > timeout)
+				return -1;
+			else
+				return normalizedTime(time);
+		}
+
+		/**
+		 * Releases all latches, and re-initializes the latches and counters.
+		 */
+		protected void releaseAndInitilize() {
+			count = 0;
+			skipLatch.countDown();
+			skipLatch = new CountDownLatch(1);
+			skipLatchUp = true;
+			stopWatch.reset();
+		}
+
+		public void reset() {
+			stopWatch.reset();
+			skipLatch.countDown();
+			count = 0;
+		}
+	}
 }
\ No newline at end of file

From 267693c9c56aa5921894392168fe652aba604bfe Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 19:06:42 +0800
Subject: [PATCH 578/881] Profiler: Initial SNProfileElement has been added.

BufferStatusData and SNProfileElement have been added.
---
 .../distributed/common/SNMessageVisitor.java  |  4 ++
 .../profiler/SNProfileElement.java            | 46 +++++++++++++++++++
 .../distributed/runtimer/StreamNodeAgent.java |  6 +++
 3 files changed, 56 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java b/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
index c4b38957..d9cebe07 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/SNMessageVisitor.java
@@ -1,5 +1,7 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement;
+
 public interface SNMessageVisitor {
 
 	void visit(Error error);
@@ -15,4 +17,6 @@ public interface SNMessageVisitor {
 	void visit(SNException snException);
 
 	void visit(SNTimeInfo timeInfo);
+
+	void visit(SNProfileElement snProfileElement);
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
new file mode 100644
index 00000000..81aea601
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
@@ -0,0 +1,46 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import com.google.common.collect.ImmutableMap;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
+import edu.mit.streamjit.impl.distributed.common.SNMessageVisitor;
+
+public abstract class SNProfileElement implements SNMessageElement {
+
+	private static final long serialVersionUID = 1L;
+
+	public abstract void process(SNProfileElementProcessor dp);
+
+	@Override
+	public void accept(SNMessageVisitor visitor) {
+		visitor.visit(this);
+	}
+
+	public static final class BufferStatusData extends SNProfileElement {
+		private static final long serialVersionUID = 1L;
+
+		public final Token blobID;
+
+		public final ImmutableMap<Token, Integer> inputBufferSizes;
+
+		public final ImmutableMap<Token, Integer> outputBufferSizes;
+
+		public BufferStatusData(Token blobID,
+				ImmutableMap<Token, Integer> inputBufferSizes,
+				ImmutableMap<Token, Integer> outputBufferSizes) {
+			this.blobID = blobID;
+			this.inputBufferSizes = inputBufferSizes;
+			this.outputBufferSizes = outputBufferSizes;
+		}
+
+		@Override
+		public void process(SNProfileElementProcessor dp) {
+			dp.process(this);
+		}
+	}
+
+	public interface SNProfileElementProcessor {
+		public void process(BufferStatusData bufferStatusData);
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index 017f7039..3d2a1016 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -22,6 +22,7 @@
 import edu.mit.streamjit.impl.distributed.common.SystemInfo;
 import edu.mit.streamjit.impl.distributed.common.SystemInfo.SystemInfoProcessor;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement;
 
 /**
  * StreamNodeAgent represents a {@link StreamNode} at {@link Controller} side.
@@ -296,5 +297,10 @@ public void visit(SNTimeInfo timeInfo) {
 			SNTimeInfoProcessor snTimeP = manager.timeInfoProcessor();
 			timeInfo.process(snTimeP);
 		}
+
+		@Override
+		public void visit(SNProfileElement snProfileElement) {
+
+		}
 	}
 }
\ No newline at end of file

From fe5d3d4b2d6a5b3d604c959651d19c5d32898f39 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 21:36:24 +0800
Subject: [PATCH 579/881] Interface StreamNodeProfiler has been added.

---
 .../profiler/StreamNodeProfiler.java          | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/StreamNodeProfiler.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/StreamNodeProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/StreamNodeProfiler.java
new file mode 100644
index 00000000..f8faf9da
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/StreamNodeProfiler.java
@@ -0,0 +1,20 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+/**
+ * Profiles a specific resources (e.g, buffer status of all blobs) of a
+ * StreamNode.
+ * 
+ * @author sumanan
+ * @since 26 Jan, 2015
+ */
+public interface StreamNodeProfiler {
+
+	/**
+	 * A profiler thread will call this method to get the current status of the
+	 * resource that is being profiled. Implementation must be thread safe.
+	 * 
+	 * @return Current status of the resource that is being profiled.
+	 */
+	public SNProfileElement profile();
+
+}
\ No newline at end of file

From fe7fb32aa466214c4f87a3203f082a708e02f861 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 26 Jan 2015 22:55:12 +0800
Subject: [PATCH 580/881] Profiler has been added

Profiler for StreamNode side.
---
 .../impl/distributed/profiler/Profiler.java   | 72 +++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
new file mode 100644
index 00000000..39886339
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -0,0 +1,72 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.collect.ImmutableSet;
+
+import edu.mit.streamjit.impl.distributed.common.Connection;
+
+public final class Profiler extends Thread {
+
+	/**
+	 * Sampling interval in ms.
+	 */
+	private final int sampleInterval = 2000;
+
+	private final ImmutableSet<StreamNodeProfiler> profilers;
+
+	private final Connection controllerConnection;
+
+	private final AtomicBoolean stopFlag;
+
+	Profiler(ImmutableSet<StreamNodeProfiler> profilers,
+			Connection controllerConnection) {
+		super("Profiler");
+		this.profilers = profilers;
+		this.controllerConnection = controllerConnection;
+		stopFlag = new AtomicBoolean(false);
+	}
+
+	public void run() {
+		while (true) {
+			sleep();
+
+			if (stopFlag.get())
+				break;
+
+			for (StreamNodeProfiler p : profilers) {
+				try {
+					controllerConnection.writeObject(p.profile());
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+
+	/**
+	 * Sleeps for t mills where (sampleInterval - 1000) < t <
+	 * (sampleInterval+1000). Because, sampling based profilers must sample at
+	 * random intervals. Not at fixed periods.
+	 */
+	private void sleep() {
+		int min = (sampleInterval - 1000);
+		int sleepTime = min + (int) (Math.random() * 2000);
+		try {
+			Thread.sleep(sleepTime);
+		} catch (InterruptedException e) {
+		}
+	}
+
+	public void stopProfiling() {
+		stopFlag.set(true);
+		this.interrupt();
+	}
+
+	public void pauseProfiling() {
+	}
+
+	public void resumeProfiling() {
+	}
+}

From ccc97aa04dc4bc7007f902ab3064d81e3ef1dae1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 17:43:03 +0800
Subject: [PATCH 581/881] ProfilerCommand has been added.

ProfilerCommnad is a CTRLRMessageElement. Controller can use these commands to
manipulate profiling operation.
---
 .../common/CTRLRMessageVisitor.java           |  4 +
 .../node/CTRLRMessageVisitorImpl.java         |  6 ++
 .../distributed/profiler/ProfilerCommand.java | 73 +++++++++++++++++++
 3 files changed, 83 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/ProfilerCommand.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/CTRLRMessageVisitor.java b/src/edu/mit/streamjit/impl/distributed/common/CTRLRMessageVisitor.java
index 9a7eeb4e..fc8fd078 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/CTRLRMessageVisitor.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/CTRLRMessageVisitor.java
@@ -1,5 +1,7 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
+
 /**
  * Visitor pattern. We have to have overloaded visit method to all sub type of
  * {@link MessageElement}s. See the {@link MessageElement}.
@@ -18,4 +20,6 @@ public interface CTRLRMessageVisitor {
 	public void visit(CTRLRDrainElement ctrlrDrainElement);
 
 	public void visit(MiscCtrlElements miscCtrlElements);
+
+	public void visit(ProfilerCommand command);
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index 78a44a04..2391cbd9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
 import edu.mit.streamjit.impl.distributed.common.Request.RequestProcessor;
+import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
 
 /**
  * @author Sumanan sumanan@mit.edu
@@ -129,4 +130,9 @@ public void processEXIT() {
 			streamNode.exit();
 		}
 	}
+
+	@Override
+	public void visit(ProfilerCommand command) {
+		// TODO Auto-generated method stub
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfilerCommand.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfilerCommand.java
new file mode 100644
index 00000000..eb454b0d
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfilerCommand.java
@@ -0,0 +1,73 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import edu.mit.streamjit.impl.distributed.common.CTRLRMessageElement;
+import edu.mit.streamjit.impl.distributed.common.CTRLRMessageVisitor;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.distributed.runtimer.Controller;
+
+/**
+ * ProfilerCommand can be send by a {@link Controller} to {@link StreamNode} to
+ * carry profiling related actions.
+ * 
+ * @author sumanan
+ * @since 27 Jan, 2015
+ */
+public enum ProfilerCommand implements CTRLRMessageElement {
+	/**
+	 * Starts the profiler.
+	 */
+	START {
+		@Override
+		public void process(ProfilerCommandProcessor commandProcessor) {
+			commandProcessor.processSTART();
+		}
+	},
+	/**
+	 * Stops the profiler.
+	 */
+	STOP {
+		@Override
+		public void process(ProfilerCommandProcessor commandProcessor) {
+			commandProcessor.processSTOP();
+		}
+	},
+
+	/**
+	 * Pause the profiler
+	 */
+	PAUSE {
+		@Override
+		public void process(ProfilerCommandProcessor commandProcessor) {
+			commandProcessor.processPAUSE();
+		}
+
+	},
+	/**
+	 * Resume the profiler
+	 */
+	RESUME {
+		@Override
+		public void process(ProfilerCommandProcessor commandProcessor) {
+			commandProcessor.processRESUME();
+		}
+	};
+
+	@Override
+	public void accept(CTRLRMessageVisitor visitor) {
+		visitor.visit(this);
+	}
+
+	public abstract void process(ProfilerCommandProcessor commandProcessor);
+
+	public interface ProfilerCommandProcessor {
+
+		public void processSTART();
+
+		public void processSTOP();
+
+		public void processPAUSE();
+
+		public void processRESUME();
+
+	}
+}

From db312670c19efb345a3b4d2c4e35cf6c2967b3ae Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 17:52:25 +0800
Subject: [PATCH 582/881] MasterProfiler added.

MasterProfiler receives profiled data from all StreamNodes and analyse the
data.
---
 .../distributed/profiler/MasterProfiler.java  | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
new file mode 100644
index 00000000..e521a32e
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -0,0 +1,28 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.BufferStatusData;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
+
+/**
+ * Profiling data from all StreamNodes come to this central point.
+ * 
+ * @author sumanan
+ * @since 27 Jan, 2015
+ */
+public class MasterProfiler implements SNProfileElementProcessor {
+
+	private final Map<Token, BufferStatusData> BufferStatusDataMap;
+
+	public MasterProfiler() {
+		BufferStatusDataMap = new HashMap<>();
+	}
+
+	@Override
+	public void process(BufferStatusData bufferStatusData) {
+		BufferStatusDataMap.put(bufferStatusData.blobID, bufferStatusData);
+	}
+}

From ee5353b9555a8301340e404fac0247ec0dd55e8d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 20:47:57 +0800
Subject: [PATCH 583/881] Renamed:BufferStatusData->SNBufferStatusData

---
 .../impl/distributed/profiler/MasterProfiler.java      |  6 +++---
 .../impl/distributed/profiler/SNProfileElement.java    | 10 +++++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index e521a32e..810c4c7c 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -4,7 +4,7 @@
 import java.util.Map;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.BufferStatusData;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
 
 /**
@@ -15,14 +15,14 @@
  */
 public class MasterProfiler implements SNProfileElementProcessor {
 
-	private final Map<Token, BufferStatusData> BufferStatusDataMap;
+	private final Map<Token, SNBufferStatusData> BufferStatusDataMap;
 
 	public MasterProfiler() {
 		BufferStatusDataMap = new HashMap<>();
 	}
 
 	@Override
-	public void process(BufferStatusData bufferStatusData) {
+	public void process(SNBufferStatusData bufferStatusData) {
 		BufferStatusDataMap.put(bufferStatusData.blobID, bufferStatusData);
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
index 81aea601..6d2098d1 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
@@ -5,6 +5,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNMessageVisitor;
+import edu.mit.streamjit.impl.distributed.node.BlobsManager;
 
 public abstract class SNProfileElement implements SNMessageElement {
 
@@ -17,7 +18,10 @@ public void accept(SNMessageVisitor visitor) {
 		visitor.visit(this);
 	}
 
-	public static final class BufferStatusData extends SNProfileElement {
+	/**
+	 * Status for all buffers from a {@link BlobsManager}.
+	 */
+	public static final class SNBufferStatusData extends SNProfileElement {
 		private static final long serialVersionUID = 1L;
 
 		public final Token blobID;
@@ -26,7 +30,7 @@ public static final class BufferStatusData extends SNProfileElement {
 
 		public final ImmutableMap<Token, Integer> outputBufferSizes;
 
-		public BufferStatusData(Token blobID,
+		public SNBufferStatusData(Token blobID,
 				ImmutableMap<Token, Integer> inputBufferSizes,
 				ImmutableMap<Token, Integer> outputBufferSizes) {
 			this.blobID = blobID;
@@ -41,6 +45,6 @@ public void process(SNProfileElementProcessor dp) {
 	}
 
 	public interface SNProfileElementProcessor {
-		public void process(BufferStatusData bufferStatusData);
+		public void process(SNBufferStatusData bufferStatusData);
 	}
 }

From 393a5011481c8076fabe0c44a124e603c8262499 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 21:41:15 +0800
Subject: [PATCH 584/881] BlobBufferStatus and BufferStatus have been added

SNBufferStatusData contains set of BlobBufferStatus where BlobBufferStatus
contains status of all buffers of a blob.
---
 .../profiler/SNProfileElement.java            | 83 ++++++++++++++++---
 1 file changed, 73 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
index 6d2098d1..11a9a1ef 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
@@ -1,6 +1,8 @@
 package edu.mit.streamjit.impl.distributed.profiler;
 
-import com.google.common.collect.ImmutableMap;
+import java.io.Serializable;
+
+import com.google.common.collect.ImmutableSet;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
@@ -22,20 +24,81 @@ public void accept(SNMessageVisitor visitor) {
 	 * Status for all buffers from a {@link BlobsManager}.
 	 */
 	public static final class SNBufferStatusData extends SNProfileElement {
+
 		private static final long serialVersionUID = 1L;
 
-		public final Token blobID;
+		public final int machineID;
+
+		public final ImmutableSet<BlobBufferStatus> blobsBufferStatusSet;
+
+		public SNBufferStatusData(int machineID,
+				ImmutableSet<BlobBufferStatus> blobsBufferStatusSet) {
+			this.machineID = machineID;
+			this.blobsBufferStatusSet = blobsBufferStatusSet;
+		}
+
+		/**
+		 * Status of all buffers of a blob.
+		 */
+		public static class BlobBufferStatus implements Serializable {
+
+			private static final long serialVersionUID = 1L;
+
+			/**
+			 * Identifier of the blob. blobID can be get through
+			 * Utils#getBlobID().
+			 */
+			public final Token blobID;
+
+			/**
+			 * BufferStatus of all input channels of the blob.
+			 */
+			public final ImmutableSet<BufferStatus> inputSet;
+
+			/**
+			 * BufferStatus of all output channels of the blob.
+			 */
+			public final ImmutableSet<BufferStatus> outputSet;
+
+			public BlobBufferStatus(Token blobID,
+					ImmutableSet<BufferStatus> inputSet,
+					ImmutableSet<BufferStatus> outputSet) {
+				this.blobID = blobID;
+				this.inputSet = inputSet;
+				this.outputSet = outputSet;
+			}
+		}
+
+		/**
+		 * Status of a single buffer.
+		 */
+		public static class BufferStatus implements Serializable {
+
+			private static final long serialVersionUID = 1L;
+
+			/**
+			 * Token of the buffer.
+			 */
+			public final Token ID;
 
-		public final ImmutableMap<Token, Integer> inputBufferSizes;
+			/**
+			 * Minimum buffer requirement. Blob.getMinimumBufferCapacity() gives
+			 * this information.
+			 */
+			public final int min;
 
-		public final ImmutableMap<Token, Integer> outputBufferSizes;
+			/**
+			 * Available resources in the buffer. If it is a input buffer then
+			 * buffer.size(). If it is a output buffer then buffer.capacity() -
+			 * buffer.size().
+			 */
+			public final int availableResource;
 
-		public SNBufferStatusData(Token blobID,
-				ImmutableMap<Token, Integer> inputBufferSizes,
-				ImmutableMap<Token, Integer> outputBufferSizes) {
-			this.blobID = blobID;
-			this.inputBufferSizes = inputBufferSizes;
-			this.outputBufferSizes = outputBufferSizes;
+			public BufferStatus(Token ID, int min, int availableResource) {
+				this.ID = ID;
+				this.min = min;
+				this.availableResource = availableResource;
+			}
 		}
 
 		@Override

From 19a264829100efc81d2844cb17d4bb0da19c66f1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 22:00:11 +0800
Subject: [PATCH 585/881] toString() has been overridden.

---
 .../impl/distributed/profiler/SNProfileElement.java   | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
index 11a9a1ef..597ae23c 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
@@ -67,6 +67,11 @@ public BlobBufferStatus(Token blobID,
 				this.inputSet = inputSet;
 				this.outputSet = outputSet;
 			}
+
+			@Override
+			public String toString() {
+				return String.format("BlobBufferStatus:blob=%s", blobID);
+			}
 		}
 
 		/**
@@ -99,6 +104,12 @@ public BufferStatus(Token ID, int min, int availableResource) {
 				this.min = min;
 				this.availableResource = availableResource;
 			}
+
+			@Override
+			public String toString() {
+				return String.format("Buffer=%s, min=%d, available=%d", ID,
+						min, availableResource);
+			}
 		}
 
 		@Override

From 19ced1e71e82816ef647bdbc759076b1f9dcc172 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 23:08:30 +0800
Subject: [PATCH 586/881] BufferProfiler has been added.

Profiles all BlobExecutors of BlobsManagerImpl and returns SNBufferStatusData.
---
 .../distributed/node/BlobsManagerImpl.java    | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index feac8cc0..f554a146 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -48,6 +48,11 @@
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.SNLocalBufferManager;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
+import edu.mit.streamjit.impl.distributed.profiler.StreamNodeProfiler;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
@@ -930,6 +935,71 @@ else if (list.size() == 1)
 		}
 	}
 
+	private class BufferProfiler {
+
+		private SNBufferStatusData snBufferStatusData() {
+			if (blobExecuters == null)
+				throw new IllegalStateException("Check this condition");
+
+			Set<BlobBufferStatus> blobBufferStatusSet = new HashSet<>();
+			for (BlobExecuter be : blobExecuters.values()) {
+				blobBufferStatusSet.add(blobBufferStatus(be));
+			}
+
+			return new SNBufferStatusData(streamNode.getNodeID(),
+					ImmutableSet.copyOf(blobBufferStatusSet));
+		}
+
+		private BlobBufferStatus blobBufferStatus(BlobExecuter be) {
+			if (be.bufferMap == null)
+				throw new IllegalStateException("Check this condition");
+
+			return new BlobBufferStatus(be.blobID, bufferStatusSet(be, true),
+					bufferStatusSet(be, false));
+		}
+
+		private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
+				boolean isIn) {
+			Set<Token> tokenSet = tokenSet(be, isIn);
+			Set<BufferStatus> bufferStatus = new HashSet<>();
+			for (Token t : tokenSet) {
+				bufferStatus.add(bufferStatus(t, be, isIn));
+			}
+			return ImmutableSet.copyOf(bufferStatus);
+		}
+
+		private BufferStatus bufferStatus(Token bufferID, BlobExecuter be,
+				boolean isIn) {
+			Buffer b = be.bufferMap.get(bufferID);
+			if (b == null)
+				throw new IllegalStateException("Check this condition");
+			int min = Integer.MAX_VALUE;
+			// BE sets blob to null after the drained().
+			if (be.blob != null)
+				min = be.blob.getMinimumBufferCapacity(bufferID);
+
+			int availableResource = isIn ? b.size() : b.capacity() - b.size();
+			return new BufferStatus(bufferID, min, availableResource);
+		}
+
+		private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
+			Set<Token> tokenSet;
+			// BE sets blob to null after the drained().
+			if (be.blob == null) {
+				if (isIn)
+					tokenSet = be.inChnlManager.inputChannelsMap().keySet();
+				else
+					tokenSet = be.outChnlManager.outputChannelsMap().keySet();
+			} else {
+				if (isIn)
+					tokenSet = be.blob.getInputs();
+				else
+					tokenSet = be.blob.getOutputs();
+			}
+			return tokenSet;
+		}
+	}
+
 	private static int count = 0;
 
 	/**

From 96803cbc09881ba010080baf3134f826af20d37a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 23:26:49 +0800
Subject: [PATCH 587/881] "blobExecuters=null" case in BufferProfiler handled

---
 .../impl/distributed/node/BlobsManagerImpl.java          | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index f554a146..0ea0c153 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -938,12 +938,11 @@ else if (list.size() == 1)
 	private class BufferProfiler {
 
 		private SNBufferStatusData snBufferStatusData() {
-			if (blobExecuters == null)
-				throw new IllegalStateException("Check this condition");
-
 			Set<BlobBufferStatus> blobBufferStatusSet = new HashSet<>();
-			for (BlobExecuter be : blobExecuters.values()) {
-				blobBufferStatusSet.add(blobBufferStatus(be));
+			if (blobExecuters == null) {
+				for (BlobExecuter be : blobExecuters.values()) {
+					blobBufferStatusSet.add(blobBufferStatus(be));
+				}
 			}
 
 			return new SNBufferStatusData(streamNode.getNodeID(),

From f34153414a74d0f87e8ef06be129dd96945982e4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 23:37:21 +0800
Subject: [PATCH 588/881] "be.bufferMap=null" case in BufferProfiler handled

---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 0ea0c153..20eb23c3 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -950,15 +950,15 @@ private SNBufferStatusData snBufferStatusData() {
 		}
 
 		private BlobBufferStatus blobBufferStatus(BlobExecuter be) {
-			if (be.bufferMap == null)
-				throw new IllegalStateException("Check this condition");
-
 			return new BlobBufferStatus(be.blobID, bufferStatusSet(be, true),
 					bufferStatusSet(be, false));
 		}
 
 		private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 				boolean isIn) {
+			if (be.bufferMap == null)
+				return ImmutableSet.of();
+
 			Set<Token> tokenSet = tokenSet(be, isIn);
 			Set<BufferStatus> bufferStatus = new HashSet<>();
 			for (Token t : tokenSet) {
@@ -966,7 +966,6 @@ private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 			}
 			return ImmutableSet.copyOf(bufferStatus);
 		}
-
 		private BufferStatus bufferStatus(Token bufferID, BlobExecuter be,
 				boolean isIn) {
 			Buffer b = be.bufferMap.get(bufferID);

From 6923ae1baba93afdf893fe4c06d1d4b35fe24e2a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 23:41:56 +0800
Subject: [PATCH 589/881] "buffer=null" case in BufferProfiler handled.

---
 .../impl/distributed/node/BlobsManagerImpl.java          | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 20eb23c3..9e370782 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -968,15 +968,16 @@ private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 		}
 		private BufferStatus bufferStatus(Token bufferID, BlobExecuter be,
 				boolean isIn) {
-			Buffer b = be.bufferMap.get(bufferID);
-			if (b == null)
-				throw new IllegalStateException("Check this condition");
 			int min = Integer.MAX_VALUE;
 			// BE sets blob to null after the drained().
 			if (be.blob != null)
 				min = be.blob.getMinimumBufferCapacity(bufferID);
 
-			int availableResource = isIn ? b.size() : b.capacity() - b.size();
+			int availableResource = min;
+			Buffer b = be.bufferMap.get(bufferID);
+			if (b != null)
+				availableResource = isIn ? b.size() : b.capacity() - b.size();
+
 			return new BufferStatus(bufferID, min, availableResource);
 		}
 

From 01c5698aa49a059c32718745fa257dd711e91945 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 27 Jan 2015 23:46:58 +0800
Subject: [PATCH 590/881] TODO added:MonitorBuffers can use BufferProfiler.

TODO: [27-01-2015] Use BufferProfiler to get buffer status and then write the
 status in to the file.
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 9e370782..c8c492ca 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -1002,6 +1002,9 @@ private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
 	private static int count = 0;
 
 	/**
+	 * TODO: [27-01-2015] Use BufferProfiler to get buffer status and then write
+	 * the status in to the file.
+	 * <p>
 	 * Profiles the buffer sizes in a timely manner and log that information
 	 * into a text file. This information may be useful to analyse and find out
 	 * deadlock situations.

From d57bc3fac77b640b0866da300877a731e8bfbc2f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 09:28:54 +0800
Subject: [PATCH 591/881] Comments have been added to BufferProfiler.

---
 .../distributed/node/BlobsManagerImpl.java    | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index c8c492ca..da6c0f4d 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -937,6 +937,10 @@ else if (list.size() == 1)
 
 	private class BufferProfiler {
 
+		/**
+		 * @return Status of all buffers of all blobs of this
+		 *         {@link BlobsManager}.
+		 */
 		private SNBufferStatusData snBufferStatusData() {
 			Set<BlobBufferStatus> blobBufferStatusSet = new HashSet<>();
 			if (blobExecuters == null) {
@@ -949,11 +953,25 @@ private SNBufferStatusData snBufferStatusData() {
 					ImmutableSet.copyOf(blobBufferStatusSet));
 		}
 
+		/**
+		 * Status of the all buffers of the blob represented by the @param be.
+		 * 
+		 * @param be
+		 * @return
+		 */
 		private BlobBufferStatus blobBufferStatus(BlobExecuter be) {
 			return new BlobBufferStatus(be.blobID, bufferStatusSet(be, true),
 					bufferStatusSet(be, false));
 		}
 
+		/**
+		 * @param be
+		 * @param isIn
+		 *            Decides whether a blob's inputbuffer's status or
+		 *            outputbuffers's status should be returned.
+		 * @return Set of {@link BufferStatus} of a blob's set of input buffers
+		 *         or set of output buffers depends on isIn argument.
+		 */
 		private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 				boolean isIn) {
 			if (be.bufferMap == null)
@@ -966,6 +984,7 @@ private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 			}
 			return ImmutableSet.copyOf(bufferStatus);
 		}
+
 		private BufferStatus bufferStatus(Token bufferID, BlobExecuter be,
 				boolean isIn) {
 			int min = Integer.MAX_VALUE;
@@ -981,6 +1000,15 @@ private BufferStatus bufferStatus(Token bufferID, BlobExecuter be,
 			return new BufferStatus(bufferID, min, availableResource);
 		}
 
+		/**
+		 * Return a blob's either input or output buffer's token set.
+		 * 
+		 * @param be
+		 * @param isIn
+		 *            Decides whether a blob's inputbuffer's token set or
+		 *            outputbuffers's token set should be returned.
+		 * @return Blob's inputbuffer's token set or outputbuffers's token set.
+		 */
 		private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
 			Set<Token> tokenSet;
 			// BE sets blob to null after the drained().
@@ -1003,7 +1031,8 @@ private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
 
 	/**
 	 * TODO: [27-01-2015] Use BufferProfiler to get buffer status and then write
-	 * the status in to the file.
+	 * the status in to the file. I created BufferProfiler by copying most of
+	 * the code from this class.
 	 * <p>
 	 * Profiles the buffer sizes in a timely manner and log that information
 	 * into a text file. This information may be useful to analyse and find out

From e707ad744e1a89ec9cd24b0fff64b1557134b88c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 09:30:56 +0800
Subject: [PATCH 592/881] Renamed:BufferProfiler->BlobsBufferStatus.

---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index da6c0f4d..af07020c 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -935,7 +935,7 @@ else if (list.size() == 1)
 		}
 	}
 
-	private class BufferProfiler {
+	private class BlobsBufferStatus {
 
 		/**
 		 * @return Status of all buffers of all blobs of this

From 873093e6764cb749af7f63341d396e86802dc9d6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 09:35:17 +0800
Subject: [PATCH 593/881] BufferProfiler implements StreamNodeProfiler added

---
 .../impl/distributed/node/BlobsManagerImpl.java        | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index af07020c..db54a749 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -1163,4 +1163,14 @@ public void stopMonitoring() {
 			this.interrupt();
 		}
 	}
+
+	public class BufferProfiler implements StreamNodeProfiler {
+
+		BlobsBufferStatus bbs = new BlobsBufferStatus();
+
+		@Override
+		public SNProfileElement profile() {
+			return bbs.snBufferStatusData();
+		}
+	}
 }

From a53e32e476567d24bd9ec394caf9ad95b1316464 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 10:08:52 +0800
Subject: [PATCH 594/881] Profiler's constructor has been made public.

---
 src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 39886339..623ca1ee 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -20,7 +20,7 @@ public final class Profiler extends Thread {
 
 	private final AtomicBoolean stopFlag;
 
-	Profiler(ImmutableSet<StreamNodeProfiler> profilers,
+	public Profiler(ImmutableSet<StreamNodeProfiler> profilers,
 			Connection controllerConnection) {
 		super("Profiler");
 		this.profilers = profilers;

From 222b9627b374d0c80645f032924275268cb3b663 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 10:10:52 +0800
Subject: [PATCH 595/881] ProfilerManager has been added

ProfilerManager implements ProfilerCommandProcessor.
---
 .../node/CTRLRMessageVisitorImpl.java         | 43 +++++++++++++++++--
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index 2391cbd9..61cfd2aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -15,7 +15,9 @@
 import edu.mit.streamjit.impl.distributed.common.NodeInfo;
 import edu.mit.streamjit.impl.distributed.common.Request;
 import edu.mit.streamjit.impl.distributed.common.Request.RequestProcessor;
+import edu.mit.streamjit.impl.distributed.profiler.Profiler;
 import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
+import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand.ProfilerCommandProcessor;
 
 /**
  * @author Sumanan sumanan@mit.edu
@@ -27,12 +29,14 @@ public class CTRLRMessageVisitorImpl implements CTRLRMessageVisitor {
 	private final RequestProcessor rp;
 	private final ConfigurationProcessor jp;
 	private final MiscCtrlElementProcessor miscProcessor;
+	private final ProfilerManager pm;
 
 	public CTRLRMessageVisitorImpl(StreamNode streamNode) {
 		this.streamNode = streamNode;
 		this.rp = new RequestProcessorImpl();
 		this.jp = new ConfigurationProcessorImpl(streamNode);
 		this.miscProcessor = new MiscCtrlElementProcessorImpl();
+		this.pm = new ProfilerManager();
 	}
 
 	@Override
@@ -73,9 +77,8 @@ public void visit(MiscCtrlElements miscCtrlElements) {
 		miscCtrlElements.process(miscProcessor);
 	}
 
-	public class MiscCtrlElementProcessorImpl
-			implements
-				MiscCtrlElementProcessor {
+	public class MiscCtrlElementProcessorImpl implements
+			MiscCtrlElementProcessor {
 
 		@Override
 		public void process(NewConInfo newConInfo) {
@@ -133,6 +136,38 @@ public void processEXIT() {
 
 	@Override
 	public void visit(ProfilerCommand command) {
-		// TODO Auto-generated method stub
+	}
+
+	public class ProfilerManager implements ProfilerCommandProcessor {
+
+		private final Profiler profiler;
+
+		ProfilerManager() {
+			profiler = null;
+		}
+
+		@Override
+		public void processSTART() {
+			// TODO Auto-generated method stub
+
+		}
+
+		@Override
+		public void processSTOP() {
+			// TODO Auto-generated method stub
+
+		}
+
+		@Override
+		public void processPAUSE() {
+			// TODO Auto-generated method stub
+
+		}
+
+		@Override
+		public void processRESUME() {
+			// TODO Auto-generated method stub
+
+		}
 	}
 }

From cf79ee60aa048a8492f8b1e35eed886df6a329be Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 11:03:52 +0800
Subject: [PATCH 596/881] StrmNdPrflr can be added to/removed from profiling

StreamNodeProfiler can be added to or removed from the Profiler.
---
 .../impl/distributed/profiler/Profiler.java   | 35 +++++++++++++++----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 623ca1ee..3fff1598 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -1,10 +1,12 @@
 package edu.mit.streamjit.impl.distributed.profiler;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import com.google.common.collect.ImmutableSet;
-
 import edu.mit.streamjit.impl.distributed.common.Connection;
 
 public final class Profiler extends Thread {
@@ -14,17 +16,21 @@ public final class Profiler extends Thread {
 	 */
 	private final int sampleInterval = 2000;
 
-	private final ImmutableSet<StreamNodeProfiler> profilers;
+	private final Set<StreamNodeProfiler> profilers;
 
 	private final Connection controllerConnection;
 
 	private final AtomicBoolean stopFlag;
 
-	public Profiler(ImmutableSet<StreamNodeProfiler> profilers,
+	public Profiler(Set<StreamNodeProfiler> profilers,
 			Connection controllerConnection) {
 		super("Profiler");
-		this.profilers = profilers;
-		this.controllerConnection = controllerConnection;
+		this.profilers = new HashSet<>();
+		checkNotNull(profilers);
+		for (StreamNodeProfiler p : profilers)
+			if (p != null)
+				profilers.add(p);
+		this.controllerConnection = checkNotNull(controllerConnection);
 		stopFlag = new AtomicBoolean(false);
 	}
 
@@ -69,4 +75,21 @@ public void pauseProfiling() {
 
 	public void resumeProfiling() {
 	}
+
+	public void addStreamNodeProfiler(StreamNodeProfiler p) {
+		checkNotNull(p, "StreamNodeProfiler is null");
+		profilers.add(p);
+	}
+
+	/**
+	 * Removes the specified StreamNodeProfiler p from profiling.
+	 * 
+	 * @param p
+	 *            StreamNodeProfiler that need to be removed from profiling.
+	 * @return <code>true</code> iff p existed in the profiler set and has been
+	 *         removed successfully.
+	 */
+	public boolean removeStreamNodeProfiler(StreamNodeProfiler p) {
+		return profilers.remove(p);
+	}
 }

From db8e77429ab26df56648d63abbe5f77ef0043256 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 11:39:51 +0800
Subject: [PATCH 597/881] ProfilerManager's methods implemented

---
 .../node/CTRLRMessageVisitorImpl.java         | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index 61cfd2aa..033a85cb 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -1,6 +1,7 @@
 package edu.mit.streamjit.impl.distributed.node;
 
 import java.io.IOException;
+import java.util.HashSet;
 
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
@@ -18,6 +19,7 @@
 import edu.mit.streamjit.impl.distributed.profiler.Profiler;
 import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
 import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand.ProfilerCommandProcessor;
+import edu.mit.streamjit.impl.distributed.profiler.StreamNodeProfiler;
 
 /**
  * @author Sumanan sumanan@mit.edu
@@ -143,31 +145,31 @@ public class ProfilerManager implements ProfilerCommandProcessor {
 		private final Profiler profiler;
 
 		ProfilerManager() {
-			profiler = null;
+			profiler = new Profiler(new HashSet<StreamNodeProfiler>(),
+					streamNode.controllerConnection);
 		}
 
 		@Override
 		public void processSTART() {
-			// TODO Auto-generated method stub
-
+			if (profiler.getState() == Thread.State.NEW)
+				profiler.start();
+			else
+				System.err.println("Profiler has already been started.");
 		}
 
 		@Override
 		public void processSTOP() {
-			// TODO Auto-generated method stub
-
+			profiler.stopProfiling();
 		}
 
 		@Override
 		public void processPAUSE() {
-			// TODO Auto-generated method stub
-
+			profiler.pauseProfiling();
 		}
 
 		@Override
 		public void processRESUME() {
-			// TODO Auto-generated method stub
-
+			profiler.resumeProfiling();
 		}
 	}
 }

From bca372222881a7bcae9731acd9def5ff8d7e143a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 11:41:37 +0800
Subject: [PATCH 598/881] Process ProfilerCommands.

---
 .../impl/distributed/node/CTRLRMessageVisitorImpl.java   | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index 033a85cb..732cb9cc 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -79,6 +79,11 @@ public void visit(MiscCtrlElements miscCtrlElements) {
 		miscCtrlElements.process(miscProcessor);
 	}
 
+	@Override
+	public void visit(ProfilerCommand command) {
+		command.process(pm);
+	}
+
 	public class MiscCtrlElementProcessorImpl implements
 			MiscCtrlElementProcessor {
 
@@ -136,10 +141,6 @@ public void processEXIT() {
 		}
 	}
 
-	@Override
-	public void visit(ProfilerCommand command) {
-	}
-
 	public class ProfilerManager implements ProfilerCommandProcessor {
 
 		private final Profiler profiler;

From 376282d2c08c6ad76fa16a94ec828e1282f6d0c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 17:47:43 +0800
Subject: [PATCH 599/881] ProfilerManager->ProfilerCommandProcessorImpl

---
 .../impl/distributed/node/CTRLRMessageVisitorImpl.java    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index 732cb9cc..fefa2a15 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -31,14 +31,14 @@ public class CTRLRMessageVisitorImpl implements CTRLRMessageVisitor {
 	private final RequestProcessor rp;
 	private final ConfigurationProcessor jp;
 	private final MiscCtrlElementProcessor miscProcessor;
-	private final ProfilerManager pm;
+	private final ProfilerCommandProcessorImpl pm;
 
 	public CTRLRMessageVisitorImpl(StreamNode streamNode) {
 		this.streamNode = streamNode;
 		this.rp = new RequestProcessorImpl();
 		this.jp = new ConfigurationProcessorImpl(streamNode);
 		this.miscProcessor = new MiscCtrlElementProcessorImpl();
-		this.pm = new ProfilerManager();
+		this.pm = new ProfilerCommandProcessorImpl();
 	}
 
 	@Override
@@ -141,11 +141,11 @@ public void processEXIT() {
 		}
 	}
 
-	public class ProfilerManager implements ProfilerCommandProcessor {
+	public class ProfilerCommandProcessorImpl implements ProfilerCommandProcessor {
 
 		private final Profiler profiler;
 
-		ProfilerManager() {
+		ProfilerCommandProcessorImpl() {
 			profiler = new Profiler(new HashSet<StreamNodeProfiler>(),
 					streamNode.controllerConnection);
 		}

From 41f1cecdc6791559c84e0339b8b660a2ab34e8c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 17:52:26 +0800
Subject: [PATCH 600/881] StreamNode keeps the profiler instance.

---
 .../node/CTRLRMessageVisitorImpl.java          | 18 +++++++++---------
 .../impl/distributed/node/StreamNode.java      |  3 +++
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index fefa2a15..b21a57c2 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -141,36 +141,36 @@ public void processEXIT() {
 		}
 	}
 
-	public class ProfilerCommandProcessorImpl implements ProfilerCommandProcessor {
-
-		private final Profiler profiler;
+	public class ProfilerCommandProcessorImpl implements
+			ProfilerCommandProcessor {
 
 		ProfilerCommandProcessorImpl() {
-			profiler = new Profiler(new HashSet<StreamNodeProfiler>(),
+			streamNode.profiler = new Profiler(
+					new HashSet<StreamNodeProfiler>(),
 					streamNode.controllerConnection);
 		}
 
 		@Override
 		public void processSTART() {
-			if (profiler.getState() == Thread.State.NEW)
-				profiler.start();
+			if (streamNode.profiler.getState() == Thread.State.NEW)
+				streamNode.profiler.start();
 			else
 				System.err.println("Profiler has already been started.");
 		}
 
 		@Override
 		public void processSTOP() {
-			profiler.stopProfiling();
+			streamNode.profiler.stopProfiling();
 		}
 
 		@Override
 		public void processPAUSE() {
-			profiler.pauseProfiling();
+			streamNode.profiler.pauseProfiling();
 		}
 
 		@Override
 		public void processRESUME() {
-			profiler.resumeProfiling();
+			streamNode.profiler.resumeProfiling();
 		}
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 32fce0fe..095a54a8 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -11,6 +11,7 @@
 import edu.mit.streamjit.impl.distributed.common.ConnectionFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Ipv4Validator;
+import edu.mit.streamjit.impl.distributed.profiler.Profiler;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
@@ -38,6 +39,8 @@ public class StreamNode extends Thread {
 
 	private volatile BlobsManager blobsManager;
 
+	Profiler profiler;
+
 	private boolean run; // As we assume that all controller communication and
 							// the MessageElement processing is managed by
 							// single

From 36f9882df2769f9895d4ea9c5fd1a0de48bcd286 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 17:54:45 +0800
Subject: [PATCH 601/881] Refactored:safelyCloseResources() added to SN.

---
 .../impl/distributed/node/StreamNode.java        | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 095a54a8..bae70795 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -92,13 +92,7 @@ public void run() {
 				run = false;
 			}
 		}
-
-		try {
-			this.controllerConnection.closeConnection();
-		} catch (IOException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
+		safelyCloseResources();
 	}
 
 	public int getNodeID() {
@@ -134,6 +128,14 @@ public String tostString() {
 		return "StreamNode-" + myNodeID;
 	}
 
+	private void safelyCloseResources() {
+		try {
+			this.controllerConnection.closeConnection();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
 	/**
 	 * @param args
 	 */

From 53e4544b095a50e8d98180da2566f9aff296676e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 17:56:37 +0800
Subject: [PATCH 602/881] safelyCloseResources() closes profiler&BlbsManager

---
 src/edu/mit/streamjit/impl/distributed/node/StreamNode.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index bae70795..cfd512ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -129,6 +129,12 @@ public String tostString() {
 	}
 
 	private void safelyCloseResources() {
+		if (blobsManager != null)
+			blobsManager.stop();
+
+		if (profiler != null)
+			profiler.stopProfiling();
+
 		try {
 			this.controllerConnection.closeConnection();
 		} catch (IOException e) {

From 5947542fc1fc770e020841b622ee4eff596a3a6b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:13:26 +0800
Subject: [PATCH 603/881] Method profilers() has been added to BlobsManager.

---
 .../streamjit/impl/distributed/node/BlobsManager.java |  8 ++++++++
 .../impl/distributed/node/BlobsManagerImpl.java       | 11 +++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
index 2a2987a4..e3f999a7 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManager.java
@@ -1,8 +1,11 @@
 package edu.mit.streamjit.impl.distributed.node;
 
+import java.util.Set;
+
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
+import edu.mit.streamjit.impl.distributed.profiler.StreamNodeProfiler;
 
 /**
  * {@link BlobsManager} is the main dispatcher for all blobs. Received commands
@@ -27,4 +30,9 @@ public interface BlobsManager {
 	 * {@link BlobsManagerImpl}#MonitorBuffers thread. Revise this.
 	 */
 	public void stop();
+
+	/**
+	 * @return Set of profilers in this BlobManager.
+	 */
+	public Set<StreamNodeProfiler> profilers();
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index db54a749..00ebbc2b 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -97,6 +97,8 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final String appName;
 
+	private ImmutableSet<StreamNodeProfiler> profilers;
+
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
 			ConnectionProvider conProvider, String appName) {
@@ -174,6 +176,15 @@ public void stop() {
 			bufferCleaner.stopit();
 	}
 
+	@Override
+	public Set<StreamNodeProfiler> profilers() {
+		if (profilers == null) {
+			StreamNodeProfiler snp = new BufferProfiler();
+			profilers = ImmutableSet.of(snp);
+		}
+		return profilers;
+	}
+
 	private void createBEs(ImmutableSet<Blob> blobSet) {
 		assert bufferManager.isbufferSizesReady() : "Buffer sizes must be available to create BlobExecuters.";
 		blobExecuters = new HashMap<>();

From 0c65c41d34a3a7ba8df1d7218120bf06e98c359a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:18:51 +0800
Subject: [PATCH 604/881] Refactored:releaseOldBM() moved to StreamNode

The method releaseOldBM() has been moved from ConfigurationProcessorImpl to
StreamNode. Because the profiler also going to refer some of the BlobsManager's
fields. So to release to be successful, those fields must also
be released. Proper place to do this operation is StreamNode.
---
 .../node/ConfigurationProcessorImpl.java         | 16 +---------------
 .../impl/distributed/node/StreamNode.java        | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index a7ca2705..f712efcd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -85,7 +85,7 @@ private void processDynamicCfg(String json, DrainData drainData) {
 		System.out
 				.println("------------------------------------------------------------");
 		System.out.println("New Configuration.....");
-		releaseOldBM();
+		streamNode.releaseOldBM();
 		Configuration cfg = Jsonifiers.fromJson(json, Configuration.class);
 		ImmutableSet<Blob> blobSet = getBlobs(cfg, drainData);
 		if (blobSet != null) {
@@ -114,20 +114,6 @@ private void processDynamicCfg(String json, DrainData drainData) {
 		}
 	}
 
-	/**
-	 * Un-references old BlobManager object before creating new one.
-	 */
-	private void releaseOldBM() {
-		// [2014-3-20] We need to release blobsmanager to release the
-		// memory. Otherwise, Blobthread2.corecode will occupy the memory.
-		BlobsManager bm = streamNode.getBlobsManager();
-		if (bm != null) {
-			bm.stop();
-			streamNode.setBlobsManager(null);
-			bm = null;
-		}
-	}
-
 	private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 			DrainData drainData) {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index cfd512ac..07b298bf 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -142,6 +142,20 @@ private void safelyCloseResources() {
 		}
 	}
 
+	/**
+	 * Un-references old BlobManager object before creating new one.
+	 */
+	void releaseOldBM() {
+		// [2014-3-20] We need to release blobsmanager to release the
+		// memory. Otherwise, Blobthread2.corecode will occupy the memory.
+		BlobsManager bm = getBlobsManager();
+		if (bm != null) {
+			bm.stop();
+			setBlobsManager(null);
+			bm = null;
+		}
+	}
+
 	/**
 	 * @param args
 	 */

From ce05d11731ad689af5b52863b1678c77669788d8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:33:58 +0800
Subject: [PATCH 605/881] Method renaming

addStreamNodeProfiler() -> add()
removeStreamNodeProfiler() -> remove()
---
 src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 3fff1598..3d0747a1 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -76,7 +76,7 @@ public void pauseProfiling() {
 	public void resumeProfiling() {
 	}
 
-	public void addStreamNodeProfiler(StreamNodeProfiler p) {
+	public void add(StreamNodeProfiler p) {
 		checkNotNull(p, "StreamNodeProfiler is null");
 		profilers.add(p);
 	}
@@ -89,7 +89,7 @@ public void addStreamNodeProfiler(StreamNodeProfiler p) {
 	 * @return <code>true</code> iff p existed in the profiler set and has been
 	 *         removed successfully.
 	 */
-	public boolean removeStreamNodeProfiler(StreamNodeProfiler p) {
+	public boolean remove(StreamNodeProfiler p) {
 		return profilers.remove(p);
 	}
 }

From 32b48742a6cf8ef38fcaf92cecea5ec9b0245e69 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:38:33 +0800
Subject: [PATCH 606/881] Profiler.removeAll(), addAll() have been added.

---
 .../impl/distributed/profiler/Profiler.java   | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 3d0747a1..36a53831 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -92,4 +92,24 @@ public void add(StreamNodeProfiler p) {
 	public boolean remove(StreamNodeProfiler p) {
 		return profilers.remove(p);
 	}
+
+	/**
+	 * Removes all profilers from profiling.
+	 * 
+	 * @param profilers
+	 */
+	public void removeAll(Set<StreamNodeProfiler> profilers) {
+		for (StreamNodeProfiler p : profilers)
+			remove(p);
+	}
+
+	/**
+	 * Add all profilers for profiling.
+	 * 
+	 * @param profilers
+	 */
+	public void addAll(Set<StreamNodeProfiler> profilers) {
+		for (StreamNodeProfiler p : profilers)
+			add(p);
+	}
 }

From 4f1381d62525af8a464a05dd52c6107e23f8d50a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:41:48 +0800
Subject: [PATCH 607/881] releaseOldBM() releases profilers() too.

---
 .../streamjit/impl/distributed/node/StreamNode.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 07b298bf..0d75bf49 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -148,11 +148,11 @@ private void safelyCloseResources() {
 	void releaseOldBM() {
 		// [2014-3-20] We need to release blobsmanager to release the
 		// memory. Otherwise, Blobthread2.corecode will occupy the memory.
-		BlobsManager bm = getBlobsManager();
-		if (bm != null) {
-			bm.stop();
-			setBlobsManager(null);
-			bm = null;
+		if (blobsManager != null) {
+			blobsManager.stop();
+			if (profiler != null)
+				profiler.removeAll(blobsManager.profilers());
+			blobsManager = null;
 		}
 	}
 

From e346ac5622c485b561d54f67848840b0867f1fcc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 22:47:00 +0800
Subject: [PATCH 608/881] setBlobsManager() releases old BlobsManager

---
 src/edu/mit/streamjit/impl/distributed/node/StreamNode.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 0d75bf49..b1fa783b 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -117,7 +117,10 @@ public BlobsManager getBlobsManager() {
 	 *            the blobsManager to set
 	 */
 	public void setBlobsManager(BlobsManager blobsManager) {
+		releaseOldBM();
 		this.blobsManager = blobsManager;
+		if (profiler != null && blobsManager != null)
+			profiler.addAll(blobsManager.profilers());
 	}
 
 	public void exit() {

From bef71c937a431bb8a5d3aa7b942ec9cc571c90f4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 28 Jan 2015 23:53:09 +0800
Subject: [PATCH 609/881] MasterProfiler.print(SNBufferStatusData) added.

---
 .../distributed/profiler/MasterProfiler.java  | 28 +++++++++++++++----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index 810c4c7c..d0ff1504 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -1,10 +1,11 @@
 package edu.mit.streamjit.impl.distributed.profiler;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
-import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
 
 /**
@@ -15,14 +16,31 @@
  */
 public class MasterProfiler implements SNProfileElementProcessor {
 
-	private final Map<Token, SNBufferStatusData> BufferStatusDataMap;
+	private final Map<Integer, SNBufferStatusData> BufferStatusDataMap;
+
+	private Object lock = new Object();
 
 	public MasterProfiler() {
-		BufferStatusDataMap = new HashMap<>();
+		BufferStatusDataMap = new ConcurrentHashMap<>();
 	}
 
 	@Override
 	public void process(SNBufferStatusData bufferStatusData) {
-		BufferStatusDataMap.put(bufferStatusData.blobID, bufferStatusData);
+		BufferStatusDataMap.put(bufferStatusData.machineID, bufferStatusData);
+		print(bufferStatusData);
+	}
+
+	private void print(SNBufferStatusData bufferStatusData) {
+		synchronized (lock) {
+			System.out.println("MachineID=" + bufferStatusData.machineID);
+			for (BlobBufferStatus bbs : bufferStatusData.blobsBufferStatusSet) {
+				System.out.println("Blob - " + bbs.blobID);
+				System.out.println("Input buffers...");
+				for (BufferStatus bs : bbs.inputSet)
+					System.out.println(bs);
+				for (BufferStatus bs : bbs.outputSet)
+					System.out.println(bs);
+			}
+		}
 	}
 }

From 12c4ab66b48e829845fc78af8dd244214c1358e6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 08:19:23 +0800
Subject: [PATCH 610/881] BugFix:blobExecuters != null.

"blobExecuters == null" statement was there. But the correct statement should
be "blobExecuters != null".
---
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 00ebbc2b..6cad6551 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -954,7 +954,7 @@ private class BlobsBufferStatus {
 		 */
 		private SNBufferStatusData snBufferStatusData() {
 			Set<BlobBufferStatus> blobBufferStatusSet = new HashSet<>();
-			if (blobExecuters == null) {
+			if (blobExecuters != null) {
 				for (BlobExecuter be : blobExecuters.values()) {
 					blobBufferStatusSet.add(blobBufferStatus(be));
 				}

From 060a0a3fed1261b1bc305cfeaa8c31f33dac5a0b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 08:55:27 +0800
Subject: [PATCH 611/881] Controller processes profile data from StreamNode

---
 .../impl/distributed/StreamJitAppManager.java          | 10 ++++++++++
 .../impl/distributed/runtimer/StreamNodeAgent.java     |  5 ++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 578268ec..cbca2301 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -43,6 +43,8 @@
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfoProcessorImpl;
 import edu.mit.streamjit.impl.distributed.common.TCPConnection.TCPConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.impl.distributed.profiler.MasterProfiler;
+import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 public class StreamJitAppManager {
@@ -65,6 +67,8 @@ public class StreamJitAppManager {
 
 	private SNExceptionProcessorImpl exP = null;
 
+	private final MasterProfiler profiler;
+
 	/**
 	 * A {@link BoundaryOutputChannel} for the head of the stream graph. If the
 	 * first {@link Worker} happened to fall outside the {@link Controller}, we
@@ -124,6 +128,7 @@ public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
 
 		headToken = Token.createOverallInputToken(app.source);
 		tailToken = Token.createOverallOutputToken(app.sink);
+		profiler = new MasterProfiler();
 	}
 
 	public AppStatusProcessor appStatusProcessor() {
@@ -349,6 +354,11 @@ private void start() {
 					tailChannel.name());
 			tailThread.start();
 		}
+		controller.sendToAll(ProfilerCommand.START);
+	}
+
+	public MasterProfiler getProfiler() {
+		return profiler;
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
index 3d2a1016..c197f092 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/StreamNodeAgent.java
@@ -23,6 +23,7 @@
 import edu.mit.streamjit.impl.distributed.common.SystemInfo.SystemInfoProcessor;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
 
 /**
  * StreamNodeAgent represents a {@link StreamNode} at {@link Controller} side.
@@ -300,7 +301,9 @@ public void visit(SNTimeInfo timeInfo) {
 
 		@Override
 		public void visit(SNProfileElement snProfileElement) {
-
+			assert manager != null : "StreamJitAppManager has not been set";
+			SNProfileElementProcessor snProfileP = manager.getProfiler();
+			snProfileElement.process(snProfileP);
 		}
 	}
 }
\ No newline at end of file

From 8f5c970b0e52d48473762fb35ebde70e6bc74d8f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 08:58:20 +0800
Subject: [PATCH 612/881] Eclipse's auto formatting changes.

---
 .../impl/distributed/StreamJitAppManager.java          | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index cbca2301..ae5156f8 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -34,8 +34,8 @@
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.Drained;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainProcessor;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNException;
 import edu.mit.streamjit.impl.distributed.common.SNException.AddressBindException;
 import edu.mit.streamjit.impl.distributed.common.SNException.SNExceptionProcessor;
@@ -328,10 +328,10 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
-		tailChannel = new TailChannels.BlockingTailChannel1(bufferMap.get(tailToken),
-				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, skipCount,
-				GlobalConstants.outputCount, app.name);
+		tailChannel = new TailChannels.BlockingTailChannel1(
+				bufferMap.get(tailToken), controller.getConProvider(),
+				tailconInfo, "tailChannel - " + tailToken.toString(), 0,
+				skipCount, GlobalConstants.outputCount, app.name);
 	}
 
 	/**

From abc1e9a7183acbec64c92e40a14132afd85cec55 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 09:23:12 +0800
Subject: [PATCH 613/881] BufferStatus.toString() prints firable info.

availableResource >= min ? "Firable":"NOT firable".
---
 .../impl/distributed/profiler/SNProfileElement.java         | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
index 597ae23c..9308674e 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/SNProfileElement.java
@@ -107,8 +107,10 @@ public BufferStatus(Token ID, int min, int availableResource) {
 
 			@Override
 			public String toString() {
-				return String.format("Buffer=%s, min=%d, available=%d", ID,
-						min, availableResource);
+				String status = availableResource >= min ? "Firable"
+						: "Not firable";
+				return String.format("Buffer=%s, min=%d, available=%d, %s", ID,
+						min, availableResource, status);
 			}
 		}
 

From 2a8d1c24fe674803b0c27ec1e0edd948b4906cab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 10:23:44 +0800
Subject: [PATCH 614/881] Interface ProfileElementLogger has been added.

---
 .../profiler/ProfileElementLogger.java            | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java
new file mode 100644
index 00000000..1e43592e
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java
@@ -0,0 +1,15 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
+
+/**
+ * Logs the {@link SNProfileElement}s. This interface extends
+ * {@link SNProfileElementProcessor} so that there is no need to manually check
+ * and add the process methods whenever a new {@link SNProfileElement} is added.
+ * 
+ * @author sumanan
+ * @since 29 Jan, 2015
+ */
+public interface ProfileElementLogger extends SNProfileElementProcessor {
+
+}

From 9298bfd972eae0ad8f8a1538963699717ee2344b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 10:53:24 +0800
Subject: [PATCH 615/881] Obsolete comments removed.

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 78dc84b1..7ea64888 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -39,10 +39,6 @@ private static FileWriter getFileWriter(String name) {
 			return fw;
 		}
 
-		/**
-		 * @param prefix
-		 *            : prefix for the file names.
-		 */
 		public FileTimeLogger(String appName) {
 			super(getFileWriter(String.format("%s%scompileTime.txt", appName,
 					File.separator)), getFileWriter(String.format(

From 83a70c6298f1f90bdf3687b40fa73175e9904a4a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:16:46 +0800
Subject: [PATCH 616/881] lock has been made final.

---
 .../mit/streamjit/impl/distributed/profiler/MasterProfiler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index d0ff1504..5243d6ba 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -18,7 +18,7 @@ public class MasterProfiler implements SNProfileElementProcessor {
 
 	private final Map<Integer, SNBufferStatusData> BufferStatusDataMap;
 
-	private Object lock = new Object();
+	private final Object lock = new Object();
 
 	public MasterProfiler() {
 		BufferStatusDataMap = new ConcurrentHashMap<>();

From 6750d590b8d73484a2cacb7e8253a693d84e8e99 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:18:43 +0800
Subject: [PATCH 617/881] Renamed:getFileWriter()->fileWriter()

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 7ea64888..798a65f9 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -29,7 +29,7 @@ public class TimeLoggers {
 	 */
 	public static class FileTimeLogger extends TimeLoggerImpl {
 
-		private static FileWriter getFileWriter(String name) {
+		private static FileWriter fileWriter(String name) {
 			FileWriter fw = null;
 			try {
 				fw = new FileWriter(name);
@@ -40,10 +40,10 @@ private static FileWriter getFileWriter(String name) {
 		}
 
 		public FileTimeLogger(String appName) {
-			super(getFileWriter(String.format("%s%scompileTime.txt", appName,
-					File.separator)), getFileWriter(String.format(
+			super(fileWriter(String.format("%s%scompileTime.txt", appName,
+					File.separator)), fileWriter(String.format(
 					"%s%srunTime.txt", appName, File.separator)),
-					getFileWriter(String.format("%s%sdrainTime.txt", appName,
+					fileWriter(String.format("%s%sdrainTime.txt", appName,
 							File.separator)));
 		}
 	}

From a46687d25511f498e271680fb5df86e52e8236f1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:27:14 +0800
Subject: [PATCH 618/881] Refactored:fileWriter() has been moved to Utils.

---
 .../impl/distributed/TimeLoggers.java         | 20 +++++--------------
 .../impl/distributed/common/Utils.java        | 18 +++++++++++++++++
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 798a65f9..4f7c4337 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -1,7 +1,6 @@
 package edu.mit.streamjit.impl.distributed;
 
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
@@ -10,6 +9,7 @@
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 
 /**
  * Collection of various {@link TimeLogger} implementations.
@@ -29,21 +29,11 @@ public class TimeLoggers {
 	 */
 	public static class FileTimeLogger extends TimeLoggerImpl {
 
-		private static FileWriter fileWriter(String name) {
-			FileWriter fw = null;
-			try {
-				fw = new FileWriter(name);
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-			return fw;
-		}
-
 		public FileTimeLogger(String appName) {
-			super(fileWriter(String.format("%s%scompileTime.txt", appName,
-					File.separator)), fileWriter(String.format(
-					"%s%srunTime.txt", appName, File.separator)),
-					fileWriter(String.format("%s%sdrainTime.txt", appName,
+			super(Utils.fileWriter(String.format("%s%scompileTime.txt",
+					appName, File.separator)), Utils.fileWriter(String.format(
+					"%s%srunTime.txt", appName, File.separator)), Utils
+					.fileWriter(String.format("%s%sdrainTime.txt", appName,
 							File.separator)));
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index fe1c07a7..02133389 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -145,4 +145,22 @@ private static void update(FileWriter writer, String name, boolean val)
 			throws IOException {
 		writer.write(String.format("%s=%s\n", name, val ? "True" : "False"));
 	}
+
+	/**
+	 * Creates and returns a {@link FileWriter} with append = false. Suppresses
+	 * {@link IOException} and returns null if exception occurred. This method
+	 * is added to keep other classes clean.
+	 * 
+	 * @param name
+	 * @return
+	 */
+	public static FileWriter fileWriter(String name) {
+		FileWriter fw = null;
+		try {
+			fw = new FileWriter(name);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return fw;
+	}
 }

From 37999c7c0740602f554d28768ee82ed995c67961 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:29:28 +0800
Subject: [PATCH 619/881] fileWriter(String name, boolean append) added.

---
 .../impl/distributed/common/Utils.java         | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 02133389..50423437 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -163,4 +163,22 @@ public static FileWriter fileWriter(String name) {
 		}
 		return fw;
 	}
+
+	/**
+	 * Creates and returns a {@link FileWriter}. Suppresses {@link IOException}
+	 * and returns null if exception occurred. This method is added to keep
+	 * other classes clean.
+	 * 
+	 * @param name
+	 * @return
+	 */
+	public static FileWriter fileWriter(String name, boolean append) {
+		FileWriter fw = null;
+		try {
+			fw = new FileWriter(name, append);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return fw;
+	}
 }

From f66eba24fcd1c87f775e2ab4e337820478558742 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:30:18 +0800
Subject: [PATCH 620/881] fileWriter(name) calls fileWriter(name,append).

---
 src/edu/mit/streamjit/impl/distributed/common/Utils.java | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 50423437..9ea5d331 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -155,13 +155,7 @@ private static void update(FileWriter writer, String name, boolean val)
 	 * @return
 	 */
 	public static FileWriter fileWriter(String name) {
-		FileWriter fw = null;
-		try {
-			fw = new FileWriter(name);
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		return fw;
+		return fileWriter(name, false);
 	}
 
 	/**

From e36773e663349bc7056c77b3f96afc25067a04df Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:33:31 +0800
Subject: [PATCH 621/881] DOTstreamVisitor uses Utils.fileWriter()

---
 .../streamjit/impl/distributed/Visualizer.java    | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index f8fbafa7..86b00287 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -109,19 +109,8 @@ private class DOTstreamVisitor extends StreamVisitor {
 			private final FileWriter writer;
 
 			DOTstreamVisitor() {
-				writer = fileWriter();
-			}
-
-			private FileWriter fileWriter() {
-				FileWriter w = null;
-				String fileName = String.format("%s%sstreamgraph.dot", appName,
-						File.separator);
-				try {
-					w = new FileWriter(fileName, false);
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-				return w;
+				writer = Utils.fileWriter(String.format("%s%sstreamgraph.dot",
+						appName, File.separator));
 			}
 
 			private void initilizeDot() {

From c21118f85442ad35e4bb02fc0a4300dbf6271418 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:36:20 +0800
Subject: [PATCH 622/881] ConfigurationPrognosticator uses Utils.fileWriter.

---
 .../runtimer/GraphPropertyPrognosticator.java    | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index bca64a19..97d0ab03 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -13,6 +13,7 @@
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
@@ -31,7 +32,8 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app) {
 		this.app = app;
-		this.writer = fileWriter();
+		this.writer = Utils.fileWriter(String.format("%s%sGraphProperty.txt",
+				app.name, File.separator));
 		writeHeader(writer);
 		paths = app.paths();
 	}
@@ -140,18 +142,6 @@ private float totalToBoundaryChannelRatio() {
 		return boundaryChannelRatio;
 	}
 
-	private FileWriter fileWriter() {
-		FileWriter w = null;
-		String fileName = String.format("%s%sGraphProperty.txt", app.name,
-				File.separator);
-		try {
-			w = new FileWriter(fileName, false);
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		return w;
-	}
-
 	private static void writeHeader(FileWriter writer) {
 		try {
 			writer.write(String.format("%.7s", "cfgID"));

From 966858d8faf61ff67552010aff586a7f04dcb931 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:41:07 +0800
Subject: [PATCH 623/881] ProfileElementLoggers have been added.

FileProfileElementLogger, PrintProfileElementLogger and
 ProfileElementLoggerImpl have been added.
---
 .../profiler/ProfileElementLoggers.java       | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100755 src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
new file mode 100755
index 00000000..899a9994
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -0,0 +1,86 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
+import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
+
+/**
+ * Collection of various {@link ProfileElementLogger} implementations.
+ * 
+ * @author sumanan
+ * @since 29 Jan, 2015
+ */
+public class ProfileElementLoggers {
+
+	public static class FileProfileElementLogger
+			extends
+				ProfileElementLoggerImpl {
+
+		public FileProfileElementLogger(OutputStream writer) {
+			super(writer);
+		}
+	}
+
+	/**
+	 * Prints the SNProfileElements to the StdOut.
+	 * 
+	 */
+	public static class PrintProfileElementLogger
+			extends
+				ProfileElementLoggerImpl {
+		public PrintProfileElementLogger() {
+			super(System.out);
+		}
+	}
+
+	private static class ProfileElementLoggerImpl implements
+			ProfileElementLogger {
+
+		private final OutputStreamWriter writer;
+
+		private final Object lock = new Object();
+
+		ProfileElementLoggerImpl(OutputStream writer) {
+			this(getOSWriter(writer));
+		}
+
+		ProfileElementLoggerImpl(OutputStreamWriter writer) {
+			this.writer = writer;
+		}
+
+		@Override
+		public void process(SNBufferStatusData bufferStatusData) {
+			if (writer == null)
+				return;
+
+			synchronized (lock) {
+				try {
+					writer.write(String.format("MachineID=%d\n",
+							bufferStatusData.machineID));
+					for (BlobBufferStatus bbs : bufferStatusData.blobsBufferStatusSet) {
+						writer.write(String.format("\tBlob=\n", bbs.blobID));
+						writer.write("\tInput buffers...\n");
+						for (BufferStatus bs : bbs.inputSet)
+							writer.write(String.format("\t\t%s\n", bs));
+						writer.write("\tOutput buffers...\n");
+						for (BufferStatus bs : bbs.outputSet)
+							writer.write(String.format("\t\t%s\n", bs));
+						writer.flush();
+					}
+
+				} catch (IOException ex) {
+				}
+			}
+		}
+
+		private static OutputStreamWriter getOSWriter(OutputStream os) {
+			if (os == null)
+				return null;
+			return new OutputStreamWriter(os);
+		}
+	}
+}

From 89e59ae46896e91d41ea026a978d2a120e2df868 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:53:04 +0800
Subject: [PATCH 624/881] FileProfileElementLogger creates fileWriter.

---
 .../impl/distributed/profiler/ProfileElementLoggers.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index 899a9994..30f4d29e 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -1,9 +1,11 @@
 package edu.mit.streamjit.impl.distributed.profiler;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
@@ -20,8 +22,9 @@ public static class FileProfileElementLogger
 			extends
 				ProfileElementLoggerImpl {
 
-		public FileProfileElementLogger(OutputStream writer) {
-			super(writer);
+		public FileProfileElementLogger(String appName) {
+			super(Utils.fileWriter(String.format("%s%sprofile.txt", appName,
+					File.separator)));
 		}
 	}
 

From 9921e80384e5f35c19bc1ed23b17d040835b9312 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 11:53:50 +0800
Subject: [PATCH 625/881] MasterProfiler logs ProfileElements.

MasterProfiler uses ProfileElementLogger to log ProfileElements.
---
 .../impl/distributed/StreamJitAppManager.java |  2 +-
 .../distributed/profiler/MasterProfiler.java  | 24 ++++---------------
 2 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index ae5156f8..8831c0ce 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -128,7 +128,7 @@ public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
 
 		headToken = Token.createOverallInputToken(app.source);
 		tailToken = Token.createOverallOutputToken(app.sink);
-		profiler = new MasterProfiler();
+		profiler = new MasterProfiler(app.name);
 	}
 
 	public AppStatusProcessor appStatusProcessor() {
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index 5243d6ba..526efcd4 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -3,9 +3,8 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import edu.mit.streamjit.impl.distributed.profiler.ProfileElementLoggers.FileProfileElementLogger;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData;
-import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
-import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNProfileElementProcessor;
 
 /**
@@ -18,29 +17,16 @@ public class MasterProfiler implements SNProfileElementProcessor {
 
 	private final Map<Integer, SNBufferStatusData> BufferStatusDataMap;
 
-	private final Object lock = new Object();
+	private final ProfileElementLogger logger;
 
-	public MasterProfiler() {
+	public MasterProfiler(String appName) {
 		BufferStatusDataMap = new ConcurrentHashMap<>();
+		logger = new FileProfileElementLogger(appName);
 	}
 
 	@Override
 	public void process(SNBufferStatusData bufferStatusData) {
 		BufferStatusDataMap.put(bufferStatusData.machineID, bufferStatusData);
-		print(bufferStatusData);
-	}
-
-	private void print(SNBufferStatusData bufferStatusData) {
-		synchronized (lock) {
-			System.out.println("MachineID=" + bufferStatusData.machineID);
-			for (BlobBufferStatus bbs : bufferStatusData.blobsBufferStatusSet) {
-				System.out.println("Blob - " + bbs.blobID);
-				System.out.println("Input buffers...");
-				for (BufferStatus bs : bbs.inputSet)
-					System.out.println(bs);
-				for (BufferStatus bs : bbs.outputSet)
-					System.out.println(bs);
-			}
-		}
+		logger.process(bufferStatusData);
 	}
 }

From a8f3feab41d47550810937287059326b30d13503 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 29 Jan 2015 12:40:19 +0800
Subject: [PATCH 626/881] BugFix: String.format().

---
 .../impl/distributed/profiler/ProfileElementLoggers.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index 30f4d29e..feb3d418 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -65,7 +65,7 @@ public void process(SNBufferStatusData bufferStatusData) {
 					writer.write(String.format("MachineID=%d\n",
 							bufferStatusData.machineID));
 					for (BlobBufferStatus bbs : bufferStatusData.blobsBufferStatusSet) {
-						writer.write(String.format("\tBlob=\n", bbs.blobID));
+						writer.write(String.format("\tBlob=%s\n", bbs.blobID));
 						writer.write("\tInput buffers...\n");
 						for (BufferStatus bs : bbs.inputSet)
 							writer.write(String.format("\t\t%s\n", bs));

From f80e96938871e43b5534f808aa881882c560dcc4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 30 Jan 2015 11:11:37 +0800
Subject: [PATCH 627/881] New method createProfiler() added.

---
 .../node/CTRLRMessageVisitorImpl.java         | 30 +++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
index b21a57c2..5da83def 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/CTRLRMessageVisitorImpl.java
@@ -145,17 +145,37 @@ public class ProfilerCommandProcessorImpl implements
 			ProfilerCommandProcessor {
 
 		ProfilerCommandProcessorImpl() {
-			streamNode.profiler = new Profiler(
-					new HashSet<StreamNodeProfiler>(),
-					streamNode.controllerConnection);
+
 		}
 
 		@Override
 		public void processSTART() {
+			createProfiler();
 			if (streamNode.profiler.getState() == Thread.State.NEW)
 				streamNode.profiler.start();
-			else
-				System.err.println("Profiler has already been started.");
+		}
+
+		/**
+		 * Creates a new profiler only if
+		 * <ol>
+		 * <li>no any profiler has already been created OR
+		 * <li>the previously created profiler has been terminated.
+		 * </ol>
+		 */
+		private void createProfiler() {
+			if (streamNode.profiler == null) {
+				streamNode.profiler = new Profiler(
+						new HashSet<StreamNodeProfiler>(),
+						streamNode.controllerConnection);
+			} else if (streamNode.profiler.getState() == Thread.State.TERMINATED) {
+				System.err
+						.println("A profiler has already been created and terminated. Creating another new profiler.");
+				streamNode.profiler = new Profiler(
+						new HashSet<StreamNodeProfiler>(),
+						streamNode.controllerConnection);
+
+			} else
+				System.err.println("A profiler has already been started.");
 		}
 
 		@Override

From 506b866a09269132628fd80b8173aca5a0e99b2b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 30 Jan 2015 12:48:57 +0800
Subject: [PATCH 628/881] GlobalConstants.needProfiler has been added.

---
 .../streamjit/impl/distributed/common/GlobalConstants.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index df4bf125..aac57f1f 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -148,7 +148,8 @@ private GlobalConstants() {
 	 */
 	public static final int maxNumCores = 24;
 
-	static {
-
-	}
+	/**
+	 * Turn On/Off the profiling.
+	 */
+	public static final boolean needProfiler = true;
 }

From 64c943a9a57c4e6514df1bd9fc9485fa18644664 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 30 Jan 2015 12:53:39 +0800
Subject: [PATCH 629/881] StreamJitAppManager.setupProfiler() added.

---
 .../impl/distributed/StreamJitAppManager.java        | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 8831c0ce..2a25e609 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -128,7 +128,7 @@ public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
 
 		headToken = Token.createOverallInputToken(app.source);
 		tailToken = Token.createOverallOutputToken(app.sink);
-		profiler = new MasterProfiler(app.name);
+		profiler = setupProfiler();
 	}
 
 	public AppStatusProcessor appStatusProcessor() {
@@ -287,6 +287,15 @@ private void reset() {
 		apStsPro.reset();
 	}
 
+	private MasterProfiler setupProfiler() {
+		MasterProfiler p = null;
+		if (GlobalConstants.needProfiler) {
+			p = new MasterProfiler(app.name);
+			controller.sendToAll(ProfilerCommand.START);
+		}
+		return p;
+	}
+
 	/**
 	 * Setup the headchannel and tailchannel.
 	 * 
@@ -354,7 +363,6 @@ private void start() {
 					tailChannel.name());
 			tailThread.start();
 		}
-		controller.sendToAll(ProfilerCommand.START);
 	}
 
 	public MasterProfiler getProfiler() {

From 43f9c85afd8d3fbd10000987999ba7e401cc5746 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 31 Jan 2015 11:53:06 +0800
Subject: [PATCH 630/881] ProfileElementLogger.newConfiguration() added.

New method newConfiguration(String cfgName) has been added to the interface ProfileElementLogger.
---
 .../distributed/profiler/ProfileElementLogger.java   |  9 +++++++++
 .../distributed/profiler/ProfileElementLoggers.java  | 12 ++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java
index 1e43592e..6bffbc79 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLogger.java
@@ -12,4 +12,13 @@
  */
 public interface ProfileElementLogger extends SNProfileElementProcessor {
 
+	/**
+	 * This method shall be called to indicate to the logger that the
+	 * configuration has been changed.
+	 * 
+	 * @param cfgName
+	 *            The name of the new configuration. Pass an empty String if the
+	 *            cfgName is not available (unknown) to the caller.
+	 */
+	public void newConfiguration(String cfgName);
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index feb3d418..7b4bdb51 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -85,5 +85,17 @@ private static OutputStreamWriter getOSWriter(OutputStream os) {
 				return null;
 			return new OutputStreamWriter(os);
 		}
+
+		@Override
+		public void newConfiguration(String cfgName) {
+			synchronized (lock) {
+				try {
+					writer.write("-----------------------------------------------------------------------------------\n");
+					writer.write(String.format("Configuration %s\n", cfgName));
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
 	}
 }

From 91fb49d5f574c0c348f98633a15e3682506c6851 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Feb 2015 21:03:27 +0800
Subject: [PATCH 631/881] MasterProfiler.logger() added.

Returns ProfileElementLogger.
---
 .../streamjit/impl/distributed/profiler/MasterProfiler.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index 526efcd4..c3d7c93e 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -19,6 +19,10 @@ public class MasterProfiler implements SNProfileElementProcessor {
 
 	private final ProfileElementLogger logger;
 
+	public ProfileElementLogger logger() {
+		return logger;
+	}
+
 	public MasterProfiler(String appName) {
 		BufferStatusDataMap = new ConcurrentHashMap<>();
 		logger = new FileProfileElementLogger(appName);

From d13efa5a2f7c54206a8ac26d8ca6509a9f88c3fa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Feb 2015 21:07:28 +0800
Subject: [PATCH 632/881] StrmJtApMngr updates PrflElmntLggr.newCfg()

StreamJitAppManager updates ProfileElementLogger.newConfiguration()
---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 2a25e609..6f251b68 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -265,6 +265,9 @@ public boolean reconfigure(int multiplier) {
 			isRunning = false;
 		}
 
+		if (profiler != null)
+			profiler.logger().newConfiguration("");
+
 		System.out.println("StraemJit app is running...");
 		Utils.printMemoryStatus();
 		return isRunning;

From 95c1e46b924c3382031753e9572f242fcc887864 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Feb 2015 08:58:56 +0800
Subject: [PATCH 633/881] Passes cfgPrefix to ProfileElementLogger.newcfg()

StreamJitAppManager passes cfgPrefix to ProfileElementLogger.newConfiguration().
This information will be written along with the profile information.
---
 .../streamjit/impl/distributed/StreamJitAppManager.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 6f251b68..c2e4a738 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -46,6 +46,7 @@
 import edu.mit.streamjit.impl.distributed.profiler.MasterProfiler;
 import edu.mit.streamjit.impl.distributed.profiler.ProfilerCommand;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
+import edu.mit.streamjit.util.ConfigurationUtils;
 
 public class StreamJitAppManager {
 
@@ -265,8 +266,10 @@ public boolean reconfigure(int multiplier) {
 			isRunning = false;
 		}
 
-		if (profiler != null)
-			profiler.logger().newConfiguration("");
+		if (profiler != null) {
+			String cfgPrefix = ConfigurationUtils.getConfigPrefix(cfg);
+			profiler.logger().newConfiguration(cfgPrefix);
+		}
 
 		System.out.println("StraemJit app is running...");
 		Utils.printMemoryStatus();

From f00c238023d7d2f6231b00eef4218f71ea0ba011 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Feb 2015 09:03:00 +0800
Subject: [PATCH 634/881] Writing format of the profile information changed.

---
 .../impl/distributed/profiler/ProfileElementLoggers.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index 7b4bdb51..ab744f63 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -90,8 +90,9 @@ private static OutputStreamWriter getOSWriter(OutputStream os) {
 		public void newConfiguration(String cfgName) {
 			synchronized (lock) {
 				try {
-					writer.write("-----------------------------------------------------------------------------------\n");
-					writer.write(String.format("Configuration %s\n", cfgName));
+					writer.write(String
+							.format("--------------------------------%s--------------------------------\n",
+									cfgName));
 				} catch (IOException e) {
 					e.printStackTrace();
 				}

From 5ebcaa5692b42693b4b0fc988d3b0e17b691203c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Feb 2015 09:29:53 +0800
Subject: [PATCH 635/881] Utils.printCurrentDateTime() has been added.

Prints current date and time in "yyyy/MM/dd HH:mm:ss" format.
---
 .../mit/streamjit/impl/distributed/common/Utils.java   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 9ea5d331..b83c2014 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -48,6 +48,7 @@ public static void printMemoryStatus() {
 		long heapFreeSize = Runtime.getRuntime().freeMemory();
 		int MEGABYTE = 1024 * 1024;
 		System.out.println("#########################");
+		printCurrentDateTime();
 		System.out.println(String.format("heapMaxSize = %dMB", heapMaxSize
 				/ MEGABYTE));
 		System.out.println(String
@@ -57,6 +58,15 @@ public static void printMemoryStatus() {
 		System.out.println("#########################");
 	}
 
+	/**
+	 * Prints current date and time in "yyyy/MM/dd HH:mm:ss" format.
+	 */
+	public static void printCurrentDateTime() {
+		DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+		Calendar cal = Calendar.getInstance();
+		System.out.println(dateFormat.format(cal.getTime()));
+	}
+
 	public static void printOutOfMemory() {
 		MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
 		System.out.println("******OutOfMemoryError******");

From 764e6b1a6a2c987d345c84b3152a1fad805d5890 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Feb 2015 23:17:14 +0800
Subject: [PATCH 636/881] BugFix: Passes correct confg to get cfgPrefix.

---
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index c2e4a738..18e67826 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -267,7 +267,8 @@ public boolean reconfigure(int multiplier) {
 		}
 
 		if (profiler != null) {
-			String cfgPrefix = ConfigurationUtils.getConfigPrefix(cfg);
+			String cfgPrefix = ConfigurationUtils.getConfigPrefix(app
+					.getConfiguration());
 			profiler.logger().newConfiguration(cfgPrefix);
 		}
 

From 8e7d13f371d77e6b771e9a5f4c8c1697a2105dbe Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Feb 2015 23:18:30 +0800
Subject: [PATCH 637/881] ProfileElementLoggers: logging format changed.

---
 .../impl/distributed/profiler/ProfileElementLoggers.java  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index ab744f63..3d60c0cd 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -66,12 +66,12 @@ public void process(SNBufferStatusData bufferStatusData) {
 							bufferStatusData.machineID));
 					for (BlobBufferStatus bbs : bufferStatusData.blobsBufferStatusSet) {
 						writer.write(String.format("\tBlob=%s\n", bbs.blobID));
-						writer.write("\tInput buffers...\n");
+						writer.write("\t\tInput...\n");
 						for (BufferStatus bs : bbs.inputSet)
-							writer.write(String.format("\t\t%s\n", bs));
-						writer.write("\tOutput buffers...\n");
+							writer.write(String.format("\t\t\t%s\n", bs));
+						writer.write("\t\tOutput...\n");
 						for (BufferStatus bs : bbs.outputSet)
-							writer.write(String.format("\t\t%s\n", bs));
+							writer.write(String.format("\t\t\t%s\n", bs));
 						writer.flush();
 					}
 

From b2eff74f1f4259883fc4dce1376c9b954772b5e1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 09:43:52 +0800
Subject: [PATCH 638/881] BlobExecuter has been moved out to a new file

Since BlobsManagerImpl grew to huge, I have refactored it and moved the inner
class BlobExecuter to a new file.
---
 .../impl/distributed/node/BlobExecuter.java   | 425 +++++++++++++++++
 .../distributed/node/BlobsManagerImpl.java    | 426 +-----------------
 2 files changed, 440 insertions(+), 411 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
new file mode 100644
index 00000000..347504cc
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -0,0 +1,425 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableTable;
+
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob;
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.blob.Buffer;
+import edu.mit.streamjit.impl.blob.Buffers;
+import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryInputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryOutputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.InputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.OutputChannelManager;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
+import edu.mit.streamjit.impl.distributed.common.Connection;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
+import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
+import edu.mit.streamjit.impl.distributed.node.BlobsManagerImpl.BlobThread2;
+import edu.mit.streamjit.impl.distributed.node.BlobsManagerImpl.DrainCallback;
+import edu.mit.streamjit.impl.distributed.runtimer.Controller;
+
+/**
+ * This class was an inner class of {@link BlobsManagerImpl}. I have re factored
+ * {@link BlobsManagerImpl} and moved this class a new file.
+ * 
+ * @author sumanan
+ * @since 4 Feb, 2015
+ */
+class BlobExecuter {
+
+	/**
+	 * 
+	 */
+	private final BlobsManagerImpl blobsManagerImpl;
+
+	Blob blob;
+
+	final Token blobID;
+
+	private Set<BlobThread2> blobThreads;
+
+	/**
+	 * Buffers for all input and output edges of the {@link #blob}.
+	 */
+	ImmutableMap<Token, Buffer> bufferMap;
+
+	private ImmutableMap<Token, LocalBuffer> outputLocalBuffers;
+
+	/**
+	 * This flag will be set to true if an exception thrown by the core code of
+	 * the {@link Blob}. Any exception occurred in a blob's corecode will be
+	 * informed to {@link Controller} to halt the application. See the
+	 * {@link BlobThread2}.
+	 */
+	AtomicBoolean crashed;
+
+	volatile int drainState;
+
+	final BoundaryInputChannelManager inChnlManager;
+
+	final BoundaryOutputChannelManager outChnlManager;
+
+	private DrainType drainType;
+
+	BlobExecuter(BlobsManagerImpl blobsManagerImpl, Token t, Blob blob,
+			ImmutableMap<Token, BoundaryInputChannel> inputChannels,
+			ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
+		this.blobsManagerImpl = blobsManagerImpl;
+		this.crashed = new AtomicBoolean(false);
+		this.blob = blob;
+		this.blobThreads = new HashSet<>();
+		assert blob.getInputs().containsAll(inputChannels.keySet());
+		assert blob.getOutputs().containsAll(outputChannels.keySet());
+		this.inChnlManager = new InputChannelManager(inputChannels);
+		this.outChnlManager = new OutputChannelManager(outputChannels);
+
+		String baseName = getName(blob);
+		for (int i = 0; i < blob.getCoreCount(); i++) {
+			String name = String.format("%s - %d", baseName, i);
+			blobThreads.add(this.blobsManagerImpl.new BlobThread2(blob
+					.getCoreCode(i), this, name));
+		}
+
+		if (blobThreads.size() < 1)
+			throw new IllegalStateException("No blobs to execute");
+
+		drainState = 0;
+		this.blobID = t;
+	}
+
+	public Token getBlobID() {
+		return blobID;
+	}
+
+	/**
+	 * Gets buffer from {@link BoundaryChannel}s and builds bufferMap. The
+	 * bufferMap will contain all input and output edges of the {@link #blob}.
+	 * 
+	 * Note that, Some {@link BoundaryChannel}s (e.g.,
+	 * {@link AsyncOutputChannel}) create {@link Buffer}s after establishing
+	 * {@link Connection} with other end. So this method must be called after
+	 * establishing all IO connections.
+	 * {@link InputChannelManager#waitToStart()} and
+	 * {@link OutputChannelManager#waitToStart()} ensure that the IO connections
+	 * are successfully established.
+	 * 
+	 * @return Buffer map which contains {@link Buffers} for all input and
+	 *         output edges of the {@link #blob}.
+	 */
+	private ImmutableMap<Token, Buffer> buildBufferMap() {
+		ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
+				.builder();
+		ImmutableMap.Builder<Token, LocalBuffer> outputLocalBufferBuilder = ImmutableMap
+				.builder();
+		ImmutableMap<Token, LocalBuffer> localBufferMap = this.blobsManagerImpl.bufferManager
+				.localBufferMap();
+		ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
+				.inputChannelsMap();
+		ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
+				.outputChannelsMap();
+
+		for (Token t : blob.getInputs()) {
+			if (localBufferMap.containsKey(t)) {
+				assert !inputChannels.containsKey(t) : "Same channels is exists in both localBuffer and inputChannel";
+				bufferMapBuilder.put(t, localBufferMap.get(t));
+			} else if (inputChannels.containsKey(t)) {
+				BoundaryInputChannel chnl = inputChannels.get(t);
+				bufferMapBuilder.put(t, chnl.getBuffer());
+			} else {
+				throw new AssertionError(String.format(
+						"No Buffer for input channel %s ", t));
+			}
+		}
+
+		for (Token t : blob.getOutputs()) {
+			if (localBufferMap.containsKey(t)) {
+				assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
+				LocalBuffer buf = localBufferMap.get(t);
+				bufferMapBuilder.put(t, buf);
+				outputLocalBufferBuilder.put(t, buf);
+			} else if (outputChannels.containsKey(t)) {
+				BoundaryOutputChannel chnl = outputChannels.get(t);
+				bufferMapBuilder.put(t, chnl.getBuffer());
+			} else {
+				throw new AssertionError(String.format(
+						"No Buffer for output channel %s ", t));
+			}
+		}
+		outputLocalBuffers = outputLocalBufferBuilder.build();
+		return bufferMapBuilder.build();
+	}
+
+	void doDrain(DrainType drainType) {
+		// System.out.println("Blob " + blobID + "is doDrain");
+		this.drainType = drainType;
+		drainState = 1;
+
+		inChnlManager.stop(drainType);
+		// TODO: [2014-03-14] I commented following line to avoid one dead
+		// lock case when draining. Deadlock 5 and 6.
+		// [2014-09-17] Lets waitToStop() if drain data is required.
+		if (drainType != DrainType.DISCARD)
+			inChnlManager.waitToStop();
+
+		for (LocalBuffer buf : outputLocalBuffers.values()) {
+			buf.drainingStarted(drainType);
+		}
+
+		if (this.blob != null) {
+			DrainCallback dcb = this.blobsManagerImpl.new DrainCallback(this);
+			drainState = 2;
+			this.blob.drain(dcb);
+		}
+		// System.out.println("Blob " + blobID +
+		// "this.blob.drain(dcb); passed");
+
+		if (this.blobsManagerImpl.useBufferCleaner
+				&& drainType != DrainType.FINAL) {
+			boolean isLastBlob = true;
+			for (BlobExecuter be : this.blobsManagerImpl.blobExecuters.values()) {
+				if (be.drainState == 0) {
+					isLastBlob = false;
+					break;
+				}
+			}
+
+			if (isLastBlob && this.blobsManagerImpl.bufferCleaner == null) {
+				System.out.println("****Starting BufferCleaner***");
+				this.blobsManagerImpl.bufferCleaner = this.blobsManagerImpl.new BufferCleaner(
+						drainType == DrainType.INTERMEDIATE);
+				this.blobsManagerImpl.bufferCleaner.start();
+			}
+		}
+	}
+
+	void drained() {
+		// System.out.println("Blob " + blobID + "drained at beg");
+		if (drainState < 3)
+			drainState = 3;
+		else
+			return;
+
+		for (BlobThread2 bt : blobThreads) {
+			bt.requestStop();
+		}
+
+		outChnlManager.stop(drainType == DrainType.FINAL);
+		outChnlManager.waitToStop();
+
+		if (drainState > 3)
+			return;
+
+		drainState = 4;
+		SNMessageElement drained = new SNDrainElement.Drained(blobID);
+		try {
+			this.blobsManagerImpl.streamNode.controllerConnection
+					.writeObject(drained);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		// System.out.println("Blob " + blobID + "is drained at mid");
+
+		if (drainType != DrainType.DISCARD) {
+			SNMessageElement me;
+			if (crashed.get())
+				me = getEmptyDrainData();
+			else
+				me = getSNDrainData();
+
+			try {
+				this.blobsManagerImpl.streamNode.controllerConnection
+						.writeObject(me);
+				// System.out.println(blobID + " DrainData has been sent");
+				drainState = 6;
+
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			// System.out.println("**********************************");
+		}
+
+		this.blob = null;
+		boolean isLastBlob = true;
+		for (BlobExecuter be : this.blobsManagerImpl.blobExecuters.values()) {
+			if (be.drainState < 4) {
+				isLastBlob = false;
+				break;
+			}
+		}
+
+		if (isLastBlob) {
+			if (this.blobsManagerImpl.monBufs != null)
+				this.blobsManagerImpl.monBufs.stopMonitoring();
+
+			if (this.blobsManagerImpl.bufferCleaner != null)
+				this.blobsManagerImpl.bufferCleaner.stopit();
+
+		}
+		// printDrainedStatus();
+	}
+
+	private SNDrainedData getSNDrainData() {
+		if (this.blob == null)
+			return getEmptyDrainData();
+
+		DrainData dd = blob.getDrainData();
+		drainState = 5;
+		// printDrainDataStats(dd);
+
+		ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
+		ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
+
+		ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
+				.inputChannelsMap();
+
+		// In a proper system the following line should be called inside
+		// doDrain(), just after inChnlManager.stop(). Read the comment
+		// in doDrain().
+		inChnlManager.waitToStop();
+
+		for (Token t : blob.getInputs()) {
+			if (inputChannels.containsKey(t)) {
+				BoundaryChannel chanl = inputChannels.get(t);
+				ImmutableList<Object> draindata = chanl.getUnprocessedData();
+				// if (draindata.size() > 0)
+				// System.out.println(String.format("From %s - %d",
+				// chanl.name(), draindata.size()));
+				inputDataBuilder.put(t, draindata);
+			}
+
+			else {
+				unprocessedDataFromLocalBuffer(inputDataBuilder, t);
+			}
+		}
+
+		ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
+				.outputChannelsMap();
+		for (Token t : blob.getOutputs()) {
+			if (outputChannels.containsKey(t)) {
+				BoundaryChannel chanl = outputChannels.get(t);
+				ImmutableList<Object> draindata = chanl.getUnprocessedData();
+				// if (draindata.size() > 0)
+				// System.out.println(String.format("From %s - %d",
+				// chanl.name(), draindata.size()));
+				outputDataBuilder.put(t, draindata);
+			}
+		}
+
+		return new SNDrainElement.SNDrainedData(blobID, dd,
+				inputDataBuilder.build(), outputDataBuilder.build());
+	}
+
+	// TODO: Unnecessary data copy. Optimise this.
+	private void unprocessedDataFromLocalBuffer(
+			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder,
+			Token t) {
+		Object[] bufArray;
+		if (this.blobsManagerImpl.bufferCleaner == null) {
+			Buffer buf = bufferMap.get(t);
+			bufArray = new Object[buf.size()];
+			buf.readAll(bufArray);
+			assert buf.size() == 0 : String.format(
+					"buffer size is %d. But 0 is expected", buf.size());
+		} else {
+			bufArray = this.blobsManagerImpl.bufferCleaner.copiedBuffer(t);
+		}
+		// if (bufArray.length > 0)
+		// System.out.println(String.format("From LocalBuffer: %s - %d",
+		// t, bufArray.length));
+		inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
+	}
+
+	private SNDrainedData getEmptyDrainData() {
+		drainState = 5;
+		ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
+		ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
+		ImmutableMap.Builder<Token, ImmutableList<Object>> dataBuilder = ImmutableMap
+				.builder();
+		ImmutableTable.Builder<Integer, String, Object> stateBuilder = ImmutableTable
+				.builder();
+		DrainData dd = new DrainData(dataBuilder.build(), stateBuilder.build());
+		return new SNDrainElement.SNDrainedData(blobID, dd,
+				inputDataBuilder.build(), outputDataBuilder.build());
+	}
+
+	/**
+	 * Returns a name for thread.
+	 * 
+	 * @param blob
+	 * @return
+	 */
+	private String getName(Blob blob) {
+		StringBuilder sb = new StringBuilder("Workers-");
+		int limit = 0;
+		for (Worker<?, ?> w : blob.getWorkers()) {
+			sb.append(Workers.getIdentifier(w));
+			sb.append(",");
+			if (++limit > 5)
+				break;
+		}
+		return sb.toString();
+	}
+
+	private void printDrainDataStats(DrainData dd) {
+		System.out.println("**********printDrainDataStats*************");
+		if (dd != null) {
+			for (Token t : dd.getData().keySet()) {
+				int size = dd.getData().get(t).size();
+				if (size > 0)
+					System.out.println("From Blob: " + t.toString() + " - "
+							+ size);
+			}
+		}
+	}
+
+	void start() {
+		outChnlManager.waitToStart();
+		inChnlManager.waitToStart();
+
+		bufferMap = buildBufferMap();
+		blob.installBuffers(bufferMap);
+
+		for (Thread t : blobThreads)
+			t.start();
+
+		System.out.println(blobID + " started");
+	}
+
+	void startChannels() {
+		outChnlManager.start();
+		inChnlManager.start();
+	}
+
+	void stop() {
+		inChnlManager.stop(DrainType.FINAL);
+		outChnlManager.stop(true);
+
+		for (Thread t : blobThreads) {
+			try {
+				t.join();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
+		inChnlManager.waitToStop();
+		outChnlManager.waitToStop();
+
+		if (this.blobsManagerImpl.monBufs != null)
+			this.blobsManagerImpl.monBufs.stopMonitoring();
+	}
+}
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 6cad6551..6776ae77 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -13,38 +13,23 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Sets;
 
-import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.blob.Buffers;
-import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryInputChannelManager;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.BoundaryOutputChannelManager;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.InputChannelManager;
-import edu.mit.streamjit.impl.distributed.common.BoundaryChannelManager.OutputChannelManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.CTRLRDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DoDrain;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainDataRequest;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
-import edu.mit.streamjit.impl.distributed.common.Connection;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
-import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
-import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.SNLocalBufferManager;
@@ -53,7 +38,6 @@
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BlobBufferStatus;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement.SNBufferStatusData.BufferStatus;
 import edu.mit.streamjit.impl.distributed.profiler.StreamNodeProfiler;
-import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
  * {@link BlobsManagerImpl} responsible to run all {@link Blob}s those are
@@ -64,9 +48,9 @@
  */
 public class BlobsManagerImpl implements BlobsManager {
 
-	private Map<Token, BlobExecuter> blobExecuters;
+	Map<Token, BlobExecuter> blobExecuters;
 
-	private final BufferManager bufferManager;
+	final BufferManager bufferManager;
 
 	private final CommandProcessor cmdProcessor;
 
@@ -74,20 +58,20 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private final ConnectionProvider conProvider;
 
-	private volatile BufferCleaner bufferCleaner = null;
+	volatile BufferCleaner bufferCleaner = null;
 
 	private final CTRLRDrainProcessor drainProcessor;
 
-	private MonitorBuffers monBufs = null;
+	MonitorBuffers monBufs = null;
 
-	private final StreamNode streamNode;
+	final StreamNode streamNode;
 
 	/**
 	 * if true {@link BufferCleaner} will be used to unlock the draining time
 	 * dead lock. Otherwise dynamic buffer will be used for local buffers to
 	 * handled drain time data growth.
 	 */
-	private final boolean useBufferCleaner = false;
+	final boolean useBufferCleaner = false;
 
 	/**
 	 * if true {@link MonitorBuffers} will be started to log the buffer sizes
@@ -197,7 +181,7 @@ private void createBEs(ImmutableSet<Blob> blobSet) {
 					Sets.difference(b.getInputs(), locaTokens), bufferSizesMap);
 			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = createOutputChannels(
 					Sets.difference(b.getOutputs(), locaTokens), bufferSizesMap);
-			blobExecuters.put(t, new BlobExecuter(t, b, inputChannels,
+			blobExecuters.put(t, new BlobExecuter(this, t, b, inputChannels,
 					outputChannels));
 		}
 	}
@@ -227,7 +211,7 @@ private ImmutableMap<Token, BoundaryOutputChannel> createOutputChannels(
 	/**
 	 * Just to added for debugging purpose.
 	 */
-	private synchronized void printDrainedStatus() {
+	synchronized void printDrainedStatus() {
 		System.out.println("****************************************");
 		for (BlobExecuter be : blobExecuters.values()) {
 			switch (be.drainState) {
@@ -265,387 +249,7 @@ private synchronized void printDrainedStatus() {
 		System.out.println("****************************************");
 	}
 
-	private class BlobExecuter {
-
-		private Blob blob;
-
-		private final Token blobID;
-
-		private Set<BlobThread2> blobThreads;
-
-		/**
-		 * Buffers for all input and output edges of the {@link #blob}.
-		 */
-		private ImmutableMap<Token, Buffer> bufferMap;
-
-		private ImmutableMap<Token, LocalBuffer> outputLocalBuffers;
-
-		/**
-		 * This flag will be set to true if an exception thrown by the core code
-		 * of the {@link Blob}. Any exception occurred in a blob's corecode will
-		 * be informed to {@link Controller} to halt the application. See the
-		 * {@link BlobThread2}.
-		 */
-		private AtomicBoolean crashed;
-
-		private volatile int drainState;
-
-		private final BoundaryInputChannelManager inChnlManager;
-
-		private final BoundaryOutputChannelManager outChnlManager;
-
-		private DrainType drainType;
-
-		private BlobExecuter(Token t, Blob blob,
-				ImmutableMap<Token, BoundaryInputChannel> inputChannels,
-				ImmutableMap<Token, BoundaryOutputChannel> outputChannels) {
-			this.crashed = new AtomicBoolean(false);
-			this.blob = blob;
-			this.blobThreads = new HashSet<>();
-			assert blob.getInputs().containsAll(inputChannels.keySet());
-			assert blob.getOutputs().containsAll(outputChannels.keySet());
-			this.inChnlManager = new InputChannelManager(inputChannels);
-			this.outChnlManager = new OutputChannelManager(outputChannels);
-
-			String baseName = getName(blob);
-			for (int i = 0; i < blob.getCoreCount(); i++) {
-				String name = String.format("%s - %d", baseName, i);
-				blobThreads
-						.add(new BlobThread2(blob.getCoreCode(i), this, name));
-			}
-
-			if (blobThreads.size() < 1)
-				throw new IllegalStateException("No blobs to execute");
-
-			drainState = 0;
-			this.blobID = t;
-		}
-
-		public Token getBlobID() {
-			return blobID;
-		}
-
-		/**
-		 * Gets buffer from {@link BoundaryChannel}s and builds bufferMap. The
-		 * bufferMap will contain all input and output edges of the
-		 * {@link #blob}.
-		 * 
-		 * Note that, Some {@link BoundaryChannel}s (e.g.,
-		 * {@link AsyncOutputChannel}) create {@link Buffer}s after establishing
-		 * {@link Connection} with other end. So this method must be called
-		 * after establishing all IO connections.
-		 * {@link InputChannelManager#waitToStart()} and
-		 * {@link OutputChannelManager#waitToStart()} ensure that the IO
-		 * connections are successfully established.
-		 * 
-		 * @return Buffer map which contains {@link Buffers} for all input and
-		 *         output edges of the {@link #blob}.
-		 */
-		private ImmutableMap<Token, Buffer> buildBufferMap() {
-			ImmutableMap.Builder<Token, Buffer> bufferMapBuilder = ImmutableMap
-					.builder();
-			ImmutableMap.Builder<Token, LocalBuffer> outputLocalBufferBuilder = ImmutableMap
-					.builder();
-			ImmutableMap<Token, LocalBuffer> localBufferMap = bufferManager
-					.localBufferMap();
-			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
-					.inputChannelsMap();
-			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
-					.outputChannelsMap();
-
-			for (Token t : blob.getInputs()) {
-				if (localBufferMap.containsKey(t)) {
-					assert !inputChannels.containsKey(t) : "Same channels is exists in both localBuffer and inputChannel";
-					bufferMapBuilder.put(t, localBufferMap.get(t));
-				} else if (inputChannels.containsKey(t)) {
-					BoundaryInputChannel chnl = inputChannels.get(t);
-					bufferMapBuilder.put(t, chnl.getBuffer());
-				} else {
-					throw new AssertionError(String.format(
-							"No Buffer for input channel %s ", t));
-				}
-			}
-
-			for (Token t : blob.getOutputs()) {
-				if (localBufferMap.containsKey(t)) {
-					assert !outputChannels.containsKey(t) : "Same channels is exists in both localBuffer and outputChannel";
-					LocalBuffer buf = localBufferMap.get(t);
-					bufferMapBuilder.put(t, buf);
-					outputLocalBufferBuilder.put(t, buf);
-				} else if (outputChannels.containsKey(t)) {
-					BoundaryOutputChannel chnl = outputChannels.get(t);
-					bufferMapBuilder.put(t, chnl.getBuffer());
-				} else {
-					throw new AssertionError(String.format(
-							"No Buffer for output channel %s ", t));
-				}
-			}
-			outputLocalBuffers = outputLocalBufferBuilder.build();
-			return bufferMapBuilder.build();
-		}
-
-		private void doDrain(DrainType drainType) {
-			// System.out.println("Blob " + blobID + "is doDrain");
-			this.drainType = drainType;
-			drainState = 1;
-
-			inChnlManager.stop(drainType);
-			// TODO: [2014-03-14] I commented following line to avoid one dead
-			// lock case when draining. Deadlock 5 and 6.
-			// [2014-09-17] Lets waitToStop() if drain data is required.
-			if (drainType != DrainType.DISCARD)
-				inChnlManager.waitToStop();
-
-			for (LocalBuffer buf : outputLocalBuffers.values()) {
-				buf.drainingStarted(drainType);
-			}
-
-			if (this.blob != null) {
-				DrainCallback dcb = new DrainCallback(this);
-				drainState = 2;
-				this.blob.drain(dcb);
-			}
-			// System.out.println("Blob " + blobID +
-			// "this.blob.drain(dcb); passed");
-
-			if (useBufferCleaner && drainType != DrainType.FINAL) {
-				boolean isLastBlob = true;
-				for (BlobExecuter be : blobExecuters.values()) {
-					if (be.drainState == 0) {
-						isLastBlob = false;
-						break;
-					}
-				}
-
-				if (isLastBlob && bufferCleaner == null) {
-					System.out.println("****Starting BufferCleaner***");
-					bufferCleaner = new BufferCleaner(
-							drainType == DrainType.INTERMEDIATE);
-					bufferCleaner.start();
-				}
-			}
-		}
-
-		private void drained() {
-			// System.out.println("Blob " + blobID + "drained at beg");
-			if (drainState < 3)
-				drainState = 3;
-			else
-				return;
-
-			for (BlobThread2 bt : blobThreads) {
-				bt.requestStop();
-			}
-
-			outChnlManager.stop(drainType == DrainType.FINAL);
-			outChnlManager.waitToStop();
-
-			if (drainState > 3)
-				return;
-
-			drainState = 4;
-			SNMessageElement drained = new SNDrainElement.Drained(blobID);
-			try {
-				streamNode.controllerConnection.writeObject(drained);
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-			// System.out.println("Blob " + blobID + "is drained at mid");
-
-			if (drainType != DrainType.DISCARD) {
-				SNMessageElement me;
-				if (crashed.get())
-					me = getEmptyDrainData();
-				else
-					me = getSNDrainData();
-
-				try {
-					streamNode.controllerConnection.writeObject(me);
-					// System.out.println(blobID + " DrainData has been sent");
-					drainState = 6;
-
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-				// System.out.println("**********************************");
-			}
-
-			this.blob = null;
-			boolean isLastBlob = true;
-			for (BlobExecuter be : blobExecuters.values()) {
-				if (be.drainState < 4) {
-					isLastBlob = false;
-					break;
-				}
-			}
-
-			if (isLastBlob) {
-				if (monBufs != null)
-					monBufs.stopMonitoring();
-
-				if (bufferCleaner != null)
-					bufferCleaner.stopit();
-
-			}
-			// printDrainedStatus();
-		}
-
-		private SNDrainedData getSNDrainData() {
-			if (this.blob == null)
-				return getEmptyDrainData();
-
-			DrainData dd = blob.getDrainData();
-			drainState = 5;
-			// printDrainDataStats(dd);
-
-			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
-			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
-
-			ImmutableMap<Token, BoundaryInputChannel> inputChannels = inChnlManager
-					.inputChannelsMap();
-
-			// In a proper system the following line should be called inside
-			// doDrain(), just after inChnlManager.stop(). Read the comment
-			// in doDrain().
-			inChnlManager.waitToStop();
-
-			for (Token t : blob.getInputs()) {
-				if (inputChannels.containsKey(t)) {
-					BoundaryChannel chanl = inputChannels.get(t);
-					ImmutableList<Object> draindata = chanl
-							.getUnprocessedData();
-					// if (draindata.size() > 0)
-					// System.out.println(String.format("From %s - %d",
-					// chanl.name(), draindata.size()));
-					inputDataBuilder.put(t, draindata);
-				}
-
-				else {
-					unprocessedDataFromLocalBuffer(inputDataBuilder, t);
-				}
-			}
-
-			ImmutableMap<Token, BoundaryOutputChannel> outputChannels = outChnlManager
-					.outputChannelsMap();
-			for (Token t : blob.getOutputs()) {
-				if (outputChannels.containsKey(t)) {
-					BoundaryChannel chanl = outputChannels.get(t);
-					ImmutableList<Object> draindata = chanl
-							.getUnprocessedData();
-					// if (draindata.size() > 0)
-					// System.out.println(String.format("From %s - %d",
-					// chanl.name(), draindata.size()));
-					outputDataBuilder.put(t, draindata);
-				}
-			}
-
-			return new SNDrainElement.SNDrainedData(blobID, dd,
-					inputDataBuilder.build(), outputDataBuilder.build());
-		}
-
-		// TODO: Unnecessary data copy. Optimise this.
-		private void unprocessedDataFromLocalBuffer(
-				ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder,
-				Token t) {
-			Object[] bufArray;
-			if (bufferCleaner == null) {
-				Buffer buf = bufferMap.get(t);
-				bufArray = new Object[buf.size()];
-				buf.readAll(bufArray);
-				assert buf.size() == 0 : String.format(
-						"buffer size is %d. But 0 is expected", buf.size());
-			} else {
-				bufArray = bufferCleaner.copiedBuffer(t);
-			}
-			// if (bufArray.length > 0)
-			// System.out.println(String.format("From LocalBuffer: %s - %d",
-			// t, bufArray.length));
-			inputDataBuilder.put(t, ImmutableList.copyOf(bufArray));
-		}
-
-		private SNDrainedData getEmptyDrainData() {
-			drainState = 5;
-			ImmutableMap.Builder<Token, ImmutableList<Object>> inputDataBuilder = new ImmutableMap.Builder<>();
-			ImmutableMap.Builder<Token, ImmutableList<Object>> outputDataBuilder = new ImmutableMap.Builder<>();
-			ImmutableMap.Builder<Token, ImmutableList<Object>> dataBuilder = ImmutableMap
-					.builder();
-			ImmutableTable.Builder<Integer, String, Object> stateBuilder = ImmutableTable
-					.builder();
-			DrainData dd = new DrainData(dataBuilder.build(),
-					stateBuilder.build());
-			return new SNDrainElement.SNDrainedData(blobID, dd,
-					inputDataBuilder.build(), outputDataBuilder.build());
-		}
-
-		/**
-		 * Returns a name for thread.
-		 * 
-		 * @param blob
-		 * @return
-		 */
-		private String getName(Blob blob) {
-			StringBuilder sb = new StringBuilder("Workers-");
-			int limit = 0;
-			for (Worker<?, ?> w : blob.getWorkers()) {
-				sb.append(Workers.getIdentifier(w));
-				sb.append(",");
-				if (++limit > 5)
-					break;
-			}
-			return sb.toString();
-		}
-
-		private void printDrainDataStats(DrainData dd) {
-			System.out.println("**********printDrainDataStats*************");
-			if (dd != null) {
-				for (Token t : dd.getData().keySet()) {
-					int size = dd.getData().get(t).size();
-					if (size > 0)
-						System.out.println("From Blob: " + t.toString() + " - "
-								+ size);
-				}
-			}
-		}
-
-		private void start() {
-			outChnlManager.waitToStart();
-			inChnlManager.waitToStart();
-
-			bufferMap = buildBufferMap();
-			blob.installBuffers(bufferMap);
-
-			for (Thread t : blobThreads)
-				t.start();
-
-			System.out.println(blobID + " started");
-		}
-
-		private void startChannels() {
-			outChnlManager.start();
-			inChnlManager.start();
-		}
-
-		private void stop() {
-			inChnlManager.stop(DrainType.FINAL);
-			outChnlManager.stop(true);
-
-			for (Thread t : blobThreads) {
-				try {
-					t.join();
-				} catch (InterruptedException e) {
-					e.printStackTrace();
-				}
-			}
-
-			inChnlManager.waitToStop();
-			outChnlManager.waitToStop();
-
-			if (monBufs != null)
-				monBufs.stopMonitoring();
-		}
-	}
-
-	private final class BlobThread2 extends Thread {
+	final class BlobThread2 extends Thread {
 
 		private final BlobExecuter be;
 
@@ -653,12 +257,12 @@ private final class BlobThread2 extends Thread {
 
 		private volatile boolean stopping = false;
 
-		private BlobThread2(Runnable coreCode, BlobExecuter be) {
+		BlobThread2(Runnable coreCode, BlobExecuter be) {
 			this.coreCode = coreCode;
 			this.be = be;
 		}
 
-		private BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
+		BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
 			super(name);
 			this.coreCode = coreCode;
 			this.be = be;
@@ -742,7 +346,7 @@ public void process(DrainDataRequest drnDataReq) {
 		}
 	}
 
-	private class DrainCallback implements Runnable {
+	class DrainCallback implements Runnable {
 
 		private final BlobExecuter blobExec;
 
@@ -789,7 +393,7 @@ public void run() {
 	 * @author sumanan
 	 * 
 	 */
-	private class BufferCleaner extends Thread {
+	class BufferCleaner extends Thread {
 
 		final AtomicBoolean run;
 
@@ -797,7 +401,7 @@ private class BufferCleaner extends Thread {
 
 		final Map<Token, List<Object[]>> newlocalBufferMap;
 
-		private BufferCleaner(boolean needToCopyDrainData) {
+		BufferCleaner(boolean needToCopyDrainData) {
 			super("BufferCleaner");
 			System.out.println("Buffer Cleaner : needToCopyDrainData == "
 					+ needToCopyDrainData);
@@ -1052,7 +656,7 @@ private Set<Token> tokenSet(BlobExecuter be, boolean isIn) {
 	 * @author sumanan
 	 * 
 	 */
-	private class MonitorBuffers extends Thread {
+	class MonitorBuffers extends Thread {
 
 		private final int id;
 

From 20d418e96c256fd3bef24740db0469d8bea810ec Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 10:37:47 +0800
Subject: [PATCH 639/881] BlobThread2 has been moved to BlobExecuter.

---
 .../impl/distributed/node/BlobExecuter.java   | 55 +++++++++++++++++--
 .../distributed/node/BlobsManagerImpl.java    | 48 ----------------
 2 files changed, 51 insertions(+), 52 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index 347504cc..fbea93cd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -16,6 +16,7 @@
 import edu.mit.streamjit.impl.blob.Buffers;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryInputChannel;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel.BoundaryOutputChannel;
@@ -28,7 +29,6 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
-import edu.mit.streamjit.impl.distributed.node.BlobsManagerImpl.BlobThread2;
 import edu.mit.streamjit.impl.distributed.node.BlobsManagerImpl.DrainCallback;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
@@ -90,8 +90,7 @@ class BlobExecuter {
 		String baseName = getName(blob);
 		for (int i = 0; i < blob.getCoreCount(); i++) {
 			String name = String.format("%s - %d", baseName, i);
-			blobThreads.add(this.blobsManagerImpl.new BlobThread2(blob
-					.getCoreCode(i), this, name));
+			blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, name));
 		}
 
 		if (blobThreads.size() < 1)
@@ -422,4 +421,52 @@ void stop() {
 		if (this.blobsManagerImpl.monBufs != null)
 			this.blobsManagerImpl.monBufs.stopMonitoring();
 	}
-}
\ No newline at end of file
+
+	final class BlobThread2 extends Thread {
+
+		private final BlobExecuter be;
+
+		private final Runnable coreCode;
+
+		private volatile boolean stopping = false;
+
+		BlobThread2(Runnable coreCode, BlobExecuter be) {
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
+			super(name);
+			this.coreCode = coreCode;
+			this.be = be;
+		}
+
+		public void requestStop() {
+			stopping = true;
+		}
+
+		@Override
+		public void run() {
+			try {
+				while (!stopping)
+					coreCode.run();
+			} catch (Error | Exception e) {
+				System.out.println(Thread.currentThread().getName()
+						+ " crashed...");
+				if (be.crashed.compareAndSet(false, true)) {
+					e.printStackTrace();
+					if (be.drainState == 1 || be.drainState == 2)
+						be.drained();
+					else if (be.drainState == 0) {
+						try {
+							blobsManagerImpl.streamNode.controllerConnection
+									.writeObject(AppStatus.ERROR);
+						} catch (IOException e1) {
+							e1.printStackTrace();
+						}
+					}
+				}
+			}
+		}
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 6776ae77..2e726b29 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -249,54 +249,6 @@ synchronized void printDrainedStatus() {
 		System.out.println("****************************************");
 	}
 
-	final class BlobThread2 extends Thread {
-
-		private final BlobExecuter be;
-
-		private final Runnable coreCode;
-
-		private volatile boolean stopping = false;
-
-		BlobThread2(Runnable coreCode, BlobExecuter be) {
-			this.coreCode = coreCode;
-			this.be = be;
-		}
-
-		BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
-			super(name);
-			this.coreCode = coreCode;
-			this.be = be;
-		}
-
-		public void requestStop() {
-			stopping = true;
-		}
-
-		@Override
-		public void run() {
-			try {
-				while (!stopping)
-					coreCode.run();
-			} catch (Error | Exception e) {
-				System.out.println(Thread.currentThread().getName()
-						+ " crashed...");
-				if (be.crashed.compareAndSet(false, true)) {
-					e.printStackTrace();
-					if (be.drainState == 1 || be.drainState == 2)
-						be.drained();
-					else if (be.drainState == 0) {
-						try {
-							streamNode.controllerConnection
-									.writeObject(AppStatus.ERROR);
-						} catch (IOException e1) {
-							e1.printStackTrace();
-						}
-					}
-				}
-			}
-		}
-	}
-
 	/**
 	 * {@link CommandProcessor} at {@link StreamNode} side.
 	 * 

From 806932cbfb16f5b33f8715a8ab4eadab3e98a76c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 10:45:33 +0800
Subject: [PATCH 640/881] DrainCallback has been moved to BlobExecutor.

---
 .../impl/distributed/node/BlobExecuter.java   | 36 +++++++++++++++++--
 .../distributed/node/BlobsManagerImpl.java    | 33 -----------------
 2 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index fbea93cd..ed74cff3 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -3,8 +3,10 @@
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableTable;
@@ -29,7 +31,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
-import edu.mit.streamjit.impl.distributed.node.BlobsManagerImpl.DrainCallback;
+import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 
 /**
@@ -179,7 +181,7 @@ void doDrain(DrainType drainType) {
 		}
 
 		if (this.blob != null) {
-			DrainCallback dcb = this.blobsManagerImpl.new DrainCallback(this);
+			DrainCallback dcb = new DrainCallback(this);
 			drainState = 2;
 			this.blob.drain(dcb);
 		}
@@ -469,4 +471,34 @@ else if (be.drainState == 0) {
 			}
 		}
 	}
+
+	class DrainCallback implements Runnable {
+
+		private final BlobExecuter blobExec;
+
+		// TODO: [2014-03-17] Just to added for checking the drain time. Remove
+		// it later.
+		private final Stopwatch sw;
+
+		DrainCallback(BlobExecuter be) {
+			this.blobExec = be;
+			sw = Stopwatch.createStarted();
+		}
+
+		@Override
+		public void run() {
+			sw.stop();
+			long time = sw.elapsed(TimeUnit.MILLISECONDS);
+			// System.out.println("Time taken to drain " + blobExec.blobID +
+			// " is " + time + " ms");
+			try {
+				blobsManagerImpl.streamNode.controllerConnection
+						.writeObject(new SNTimeInfo.DrainingTime(
+								blobExec.blobID, time));
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			blobExec.drained();
+		}
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 2e726b29..a4416d1f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -9,10 +9,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
@@ -30,7 +28,6 @@
 import edu.mit.streamjit.impl.distributed.common.Command.CommandProcessor;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
-import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BufferManager.SNLocalBufferManager;
 import edu.mit.streamjit.impl.distributed.profiler.SNProfileElement;
@@ -298,36 +295,6 @@ public void process(DrainDataRequest drnDataReq) {
 		}
 	}
 
-	class DrainCallback implements Runnable {
-
-		private final BlobExecuter blobExec;
-
-		// TODO: [2014-03-17] Just to added for checking the drain time. Remove
-		// it later.
-		private final Stopwatch sw;
-
-		DrainCallback(BlobExecuter be) {
-			this.blobExec = be;
-			sw = Stopwatch.createStarted();
-		}
-
-		@Override
-		public void run() {
-			sw.stop();
-			long time = sw.elapsed(TimeUnit.MILLISECONDS);
-			// System.out.println("Time taken to drain " + blobExec.blobID +
-			// " is " + time + " ms");
-			try {
-				streamNode.controllerConnection
-						.writeObject(new SNTimeInfo.DrainingTime(
-								blobExec.blobID, time));
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-			blobExec.drained();
-		}
-	}
-
 	/**
 	 * Handles another type of deadlock which occurs when draining. A Down blob,
 	 * that has more than one upper blob, cannot progress because some of its

From ae5e14b3ee8ab212cf48fb552fa1f4a47bf24958 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 13:53:14 +0800
Subject: [PATCH 641/881] Field BlobExecuter.blobThreads has been made final

---
 src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index ed74cff3..35af0976 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -52,7 +52,7 @@ class BlobExecuter {
 
 	final Token blobID;
 
-	private Set<BlobThread2> blobThreads;
+	final private Set<BlobThread2> blobThreads;
 
 	/**
 	 * Buffers for all input and output edges of the {@link #blob}.

From 2ee2e0aab964b37249b7c9afe1fcb995a7972fae Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 14:09:04 +0800
Subject: [PATCH 642/881] New interface AffinityManager has been added.

---
 .../distributed/node/AffinityManager.java     | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/AffinityManager.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/AffinityManager.java b/src/edu/mit/streamjit/impl/distributed/node/AffinityManager.java
new file mode 100644
index 00000000..4a6ca579
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/AffinityManager.java
@@ -0,0 +1,24 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import com.google.common.collect.ImmutableSet;
+
+import edu.mit.streamjit.impl.blob.Blob;
+import edu.mit.streamjit.impl.distributed.node.BlobExecuter.BlobThread2;
+
+/**
+ * Assigns CPU cores to {@link BlobThread2}s. {@link BlobThread2}s are expected
+ * to set their processor affinity which is given by
+ * {@link AffinityManager#getAffinity(Blob, int)} before start running.
+ * 
+ * @author sumanan
+ * @since 4 Feb, 2015
+ */
+public interface AffinityManager {
+
+	/**
+	 * @param blob
+	 * @param coreCode
+	 * @return Set of CPU cores that is assigned the blob's coreCode.
+	 */
+	ImmutableSet<Integer> getAffinity(Blob blob, int coreCode);
+}

From f00f9eeb182e37809c2c8ae125994ea4d569681b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 14:16:39 +0800
Subject: [PATCH 643/881] Unused BlobThread2's constructor removed.

---
 .../mit/streamjit/impl/distributed/node/BlobExecuter.java    | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index 35af0976..ccb20551 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -432,11 +432,6 @@ final class BlobThread2 extends Thread {
 
 		private volatile boolean stopping = false;
 
-		BlobThread2(Runnable coreCode, BlobExecuter be) {
-			this.coreCode = coreCode;
-			this.be = be;
-		}
-
 		BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
 			super(name);
 			this.coreCode = coreCode;

From 65ff4e8b7929a0592327548218239493575007ae Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 14:24:02 +0800
Subject: [PATCH 644/881] BlobThread2 sets processor affinity.

---
 .../impl/distributed/node/BlobExecuter.java         | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index ccb20551..d4bfef84 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -33,6 +33,7 @@
 import edu.mit.streamjit.impl.distributed.common.SNMessageElement;
 import edu.mit.streamjit.impl.distributed.common.SNTimeInfo;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
+import edu.mit.streamjit.util.affinity.Affinity;
 
 /**
  * This class was an inner class of {@link BlobsManagerImpl}. I have re factored
@@ -92,7 +93,8 @@ class BlobExecuter {
 		String baseName = getName(blob);
 		for (int i = 0; i < blob.getCoreCount(); i++) {
 			String name = String.format("%s - %d", baseName, i);
-			blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, name));
+			blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, name,
+					null));
 		}
 
 		if (blobThreads.size() < 1)
@@ -426,16 +428,20 @@ void stop() {
 
 	final class BlobThread2 extends Thread {
 
+		private final Set<Integer> cores;
+
 		private final BlobExecuter be;
 
 		private final Runnable coreCode;
 
 		private volatile boolean stopping = false;
 
-		BlobThread2(Runnable coreCode, BlobExecuter be, String name) {
+		BlobThread2(Runnable coreCode, BlobExecuter be, String name,
+				Set<Integer> cores) {
 			super(name);
 			this.coreCode = coreCode;
 			this.be = be;
+			this.cores = cores;
 		}
 
 		public void requestStop() {
@@ -444,6 +450,9 @@ public void requestStop() {
 
 		@Override
 		public void run() {
+			if (cores != null && cores.size() > 0)
+				Affinity.setThreadAffinity(cores);
+
 			try {
 				while (!stopping)
 					coreCode.run();

From 69f76997371fb4a5efde43caf69810951091cfd2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 14:47:27 +0800
Subject: [PATCH 645/881] AffinityManagers have been added.

Various implementations of the interface AffinityManager.
---
 .../distributed/node/AffinityManagers.java    | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/node/AffinityManagers.java

diff --git a/src/edu/mit/streamjit/impl/distributed/node/AffinityManagers.java b/src/edu/mit/streamjit/impl/distributed/node/AffinityManagers.java
new file mode 100644
index 00000000..7d19f705
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/node/AffinityManagers.java
@@ -0,0 +1,29 @@
+package edu.mit.streamjit.impl.distributed.node;
+
+import com.google.common.collect.ImmutableSet;
+
+import edu.mit.streamjit.impl.blob.Blob;
+
+/**
+ * Various implementations of the interface {@link AffinityManager}.
+ * 
+ * @author sumanan
+ * @since 4 Feb, 2015
+ */
+public class AffinityManagers {
+
+	/**
+	 * This is an empty {@link AffinityManager}. {@link #getAffinity(Blob, int)}
+	 * always returns null.
+	 * 
+	 * @author sumanan
+	 * @since 4 Feb, 2015
+	 */
+	public static class EmptyAffinityManager implements AffinityManager {
+
+		@Override
+		public ImmutableSet<Integer> getAffinity(Blob blob, int coreCode) {
+			return null;
+		}
+	}
+}

From f641317fb23b74dcf1791ec565dd5468bec07b51 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Feb 2015 14:50:09 +0800
Subject: [PATCH 646/881] BlobsManagerImpl keeps AffinityManager

BlobExecuter uses BlobsManagerImpl.affinityManager to getAffinity for each
thread.
---
 src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java  | 2 +-
 .../mit/streamjit/impl/distributed/node/BlobsManagerImpl.java  | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
index d4bfef84..95f27428 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobExecuter.java
@@ -94,7 +94,7 @@ class BlobExecuter {
 		for (int i = 0; i < blob.getCoreCount(); i++) {
 			String name = String.format("%s - %d", baseName, i);
 			blobThreads.add(new BlobThread2(blob.getCoreCode(i), this, name,
-					null));
+					blobsManagerImpl.affinityManager.getAffinity(blob, i)));
 		}
 
 		if (blobThreads.size() < 1)
diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index a4416d1f..7a47f6a4 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -80,6 +80,8 @@ public class BlobsManagerImpl implements BlobsManager {
 
 	private ImmutableSet<StreamNodeProfiler> profilers;
 
+	final AffinityManager affinityManager;
+
 	public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 			Map<Token, ConnectionInfo> conInfoMap, StreamNode streamNode,
 			ConnectionProvider conProvider, String appName) {
@@ -90,6 +92,7 @@ public BlobsManagerImpl(ImmutableSet<Blob> blobSet,
 		this.cmdProcessor = new CommandProcessorImpl();
 		this.drainProcessor = new CTRLRDrainProcessorImpl();
 		this.bufferManager = new SNLocalBufferManager(blobSet);
+		this.affinityManager = new AffinityManagers.EmptyAffinityManager();
 
 		this.appName = appName;
 		bufferManager.initialise();

From d1afebcb2b2617031f37b14f13dbe14be73d694c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Feb 2015 09:00:30 +0800
Subject: [PATCH 647/881] StreamNode joins Profiler thread b4 terminate

When safelyCloseResources(), StreamNode thread calls Profiler thread.join().
---
 src/edu/mit/streamjit/impl/distributed/node/StreamNode.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index b1fa783b..8bf3ff27 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -138,6 +138,12 @@ private void safelyCloseResources() {
 		if (profiler != null)
 			profiler.stopProfiling();
 
+		try {
+			profiler.join();
+		} catch (InterruptedException e1) {
+			e1.printStackTrace();
+		}
+
 		try {
 			this.controllerConnection.closeConnection();
 		} catch (IOException e) {

From 7b10d7f7d95d9b872935df7b24e00ab0683bdd7f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Feb 2015 15:07:25 +0800
Subject: [PATCH 648/881] addBufferSize() moved to super class.

---
 .../impl/distributed/node/BufferManager.java  | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
index a6ff4c64..827cb80f 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BufferManager.java
@@ -158,6 +158,18 @@ protected final LocalBuffer concurrentArrayLocalBuffer(Token t,
 				int bufSize) {
 			return new ConcurrentArrayLocalBuffer(bufSize);
 		}
+
+		/**
+		 * Just introduced to avoid code duplication.
+		 */
+		protected void addBufferSize(Token t, int minSize,
+				ImmutableMap.Builder<Token, Integer> bufferSizeMapBuilder) {
+			// TODO: Just to increase the performance. Change it later
+			int bufSize = Math.max(1000, minSize);
+			// System.out.println("Buffer size of " + t.toString() + " is " +
+			// bufSize);
+			bufferSizeMapBuilder.put(t, bufSize);
+		}
 	}
 
 	/**
@@ -228,17 +240,5 @@ private ImmutableMap<Token, Integer> calculateBufferSizes(
 			}
 			return bufferSizeMapBuilder.build();
 		}
-
-		/**
-		 * Just introduced to avoid code duplication.
-		 */
-		private void addBufferSize(Token t, int minSize,
-				ImmutableMap.Builder<Token, Integer> bufferSizeMapBuilder) {
-			// TODO: Just to increase the performance. Change it later
-			int bufSize = Math.max(1000, minSize);
-			// System.out.println("Buffer size of " + t.toString() + " is " +
-			// bufSize);
-			bufferSizeMapBuilder.put(t, bufSize);
-		}
 	}
 }

From 4fd1a34832ae17265f80cfd2ed84119437034af9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Feb 2015 13:52:12 +0800
Subject: [PATCH 649/881] OnlineTuner.verifyTuningTimes() handles null cfgs.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 4 ++++
 src/edu/mit/streamjit/util/ConfigurationUtils.java            | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 6e4b713e..b71a6c3f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -173,6 +173,10 @@ private void verifyTuningTimes() {
 			String cfgName = String.format("%s_%s.cfg", prefix, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
 					prefix);
+			if (cfg == null) {
+				System.err.println(String.format("No %s file exists", cfgName));
+				continue;
+			}
 			evaluateConfig(cfg, cfgName);
 		}
 		terminate();
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 24019ce9..0ea6102c 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -29,6 +29,9 @@ public class ConfigurationUtils {
 	 * 
 	 * @param namePrefix
 	 *            prefix to add to the cfg file name.
+	 * 
+	 * @return {@link Configuration} object if valid file exists. Otherwise
+	 *         returns null.
 	 */
 	public static Configuration readConfiguration(String appName,
 			String namePrefix) {

From 288b5f6d5512d7789b594396b1752b5527d40817 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Feb 2015 13:58:07 +0800
Subject: [PATCH 650/881] verifyTuningTimes() adds cfgPrefix.

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b71a6c3f..6b353092 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -177,6 +177,7 @@ private void verifyTuningTimes() {
 				System.err.println(String.format("No %s file exists", cfgName));
 				continue;
 			}
+			cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
 			evaluateConfig(cfg, cfgName);
 		}
 		terminate();

From 2437f885ac10ea08e138d3a2887cebac988f6bbd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 08:48:57 +0800
Subject: [PATCH 651/881] Null checks.

---
 src/edu/mit/streamjit/util/ConfigurationUtils.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 0ea6102c..51b4525a 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -135,6 +135,8 @@ public static void saveConfg(String configJson, String namePrefix,
 	 */
 	public static Configuration addConfigPrefix(Configuration config,
 			String prefix) {
+		if (config == null)
+			return config;
 		Configuration.Builder builder = Configuration.builder(config);
 		builder.putExtraData("configPrefix", prefix);
 		return builder.build();
@@ -148,6 +150,8 @@ public static Configuration addConfigPrefix(Configuration config,
 	 *         otherwise.
 	 */
 	public static String getConfigPrefix(Configuration config) {
+		if (config == null)
+			return null;
 		String prefix = (String) config.getExtraData("configPrefix");
 		return prefix == null ? "" : prefix;
 	}

From f541693cc396797a6961da2f01617ccb6fc0c930 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 08:59:00 +0800
Subject: [PATCH 652/881] Checks profiler's status for null before join().

---
 .../streamjit/impl/distributed/node/StreamNode.java   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 8bf3ff27..86c77c60 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -135,13 +135,14 @@ private void safelyCloseResources() {
 		if (blobsManager != null)
 			blobsManager.stop();
 
-		if (profiler != null)
+		if (profiler != null) {
 			profiler.stopProfiling();
 
-		try {
-			profiler.join();
-		} catch (InterruptedException e1) {
-			e1.printStackTrace();
+			try {
+				profiler.join();
+			} catch (InterruptedException e1) {
+				e1.printStackTrace();
+			}
 		}
 
 		try {

From 813f67b06475c9dd526107318837d68e78217e7a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 09:35:46 +0800
Subject: [PATCH 653/881] Profiler thread exits if IOExp occured.

---
 src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 36a53831..0f61027e 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -46,9 +46,11 @@ public void run() {
 					controllerConnection.writeObject(p.profile());
 				} catch (IOException e) {
 					e.printStackTrace();
+					stopFlag.set(true);
 				}
 			}
 		}
+		System.err.println("Profiler is exiting");
 	}
 
 	/**

From 1ef4bbb6fe7128853b5a3ac67e20a23458b6d4f4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 09:37:15 +0800
Subject: [PATCH 654/881] Checks isStillConnected() before try to close con

StreamNode checks controllerConnection.isStillConnected() status before trying
to close the controller connection.
---
 .../streamjit/impl/distributed/node/StreamNode.java   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
index 86c77c60..9e7aa9dd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/StreamNode.java
@@ -145,11 +145,12 @@ private void safelyCloseResources() {
 			}
 		}
 
-		try {
-			this.controllerConnection.closeConnection();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
+		if (controllerConnection.isStillConnected())
+			try {
+				this.controllerConnection.closeConnection();
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
 	}
 
 	/**

From 54693f419d4e9b4165f03d55b72803a86ab705a9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 10:41:43 +0800
Subject: [PATCH 655/881] e.printStackTrace() commented at Profiler.

---
 src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
index 0f61027e..78507d06 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/Profiler.java
@@ -45,7 +45,7 @@ public void run() {
 				try {
 					controllerConnection.writeObject(p.profile());
 				} catch (IOException e) {
-					e.printStackTrace();
+					// e.printStackTrace();
 					stopFlag.set(true);
 				}
 			}

From e41dd08c4b2d76a0906c9662206fbcf7a5554e41 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Feb 2015 22:40:26 +0800
Subject: [PATCH 656/881] Stops sending profile data after draining.

"be.blob == null" condition is added to avoid sending profile data after the
blob has been drained.
---
 .../streamjit/impl/distributed/node/BlobsManagerImpl.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
index 7a47f6a4..f779dae3 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlobsManagerImpl.java
@@ -511,7 +511,11 @@ private BlobBufferStatus blobBufferStatus(BlobExecuter be) {
 		 */
 		private ImmutableSet<BufferStatus> bufferStatusSet(BlobExecuter be,
 				boolean isIn) {
-			if (be.bufferMap == null)
+			// TODO: [Feb 8, 2015] "be.blob == null" condition is added to
+			// avoid sending profile data after the blob has been drained. But
+			// we may need the "after draining buffer status" when analyzing
+			// dead lock situations. Remove "be.blob == null" at that time.
+			if (be.bufferMap == null || be.blob == null)
 				return ImmutableSet.of();
 
 			Set<Token> tokenSet = tokenSet(be, isIn);

From 866fb5a4d268f126ee9d167ac4e95829078948ca Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Feb 2015 11:36:55 +0800
Subject: [PATCH 657/881] reconfigure() returns various negative values

OnlineTuner.reconfigure() returns various negative values based on the different
failtures.
---
 .../distributed/runtimer/OnlineTuner.java     | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 6b353092..e0bad921 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -13,6 +13,7 @@
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
 import edu.mit.streamjit.util.ConfigurationUtils;
@@ -190,7 +191,18 @@ private void verifyTuningTimes() {
 	 * @param cfgJson
 	 * @param round
 	 * @return if ret.first == false, then no more tuning. ret.second = running
-	 *         time in milliseconds.
+	 *         time in milliseconds. ret.second may be a negative value if the
+	 *         reconfiguration is unsuccessful or a timeout is occurred.
+	 *         Meanings of the negative values are follows
+	 *         <ol>
+	 *         <li>-1: Timeout has occurred.
+	 *         <li>-2: Invalid configuration.
+	 *         <li>-3: {@link ConfigurationPrognosticator} has rejected the
+	 *         configuration.
+	 *         <li>-4: Draining failed. Another draining is in progress.
+	 *         <li>-5: Reconfiguration has failed at {@link StreamNode} side.
+	 *         E.g., Compilation error.
+	 *         <li>-6: Misc problems.
 	 */
 	private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		long time;
@@ -200,13 +212,13 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 
 		try {
 			if (!cfgManager.newConfiguration(config))
-				return new Pair<Boolean, Long>(true, -1l);
+				return new Pair<Boolean, Long>(true, -2l);
 
 			if (!prognosticator.prognosticate(config))
-				return new Pair<Boolean, Long>(true, -1l);
+				return new Pair<Boolean, Long>(true, -3l);
 
 			if (!intermediateDraining())
-				return new Pair<Boolean, Long>(false, -1l);
+				return new Pair<Boolean, Long>(false, -4l);
 
 			drainer.setBlobGraph(app.blobGraph);
 			int multiplier = getMultiplier(config);
@@ -217,13 +229,13 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 				time = manager.getFixedOutputTime(timeout);
 				logger.logRunTime(time);
 			} else {
-				time = -1l;
+				time = -5l;
 			}
 		} catch (Exception ex) {
 			ex.printStackTrace();
 			System.err
 					.println("Couldn't compile the stream graph with this configuration");
-			time = -1l;
+			time = -6l;
 		}
 		return new Pair<Boolean, Long>(true, time);
 	}

From 134f65f6407dbe9a511825c675552953aaff29ab Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Feb 2015 11:41:35 +0800
Subject: [PATCH 658/881] OnlineTuner: Codes moved out from try-catch block

Codes which do not throw exceptions have been moved out from the try-catch
block.
---
 .../impl/distributed/runtimer/OnlineTuner.java         | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e0bad921..71bc2dc1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -210,13 +210,13 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		if (manager.getStatus() == AppStatus.STOPPED)
 			return new Pair<Boolean, Long>(false, 0l);
 
-		try {
-			if (!cfgManager.newConfiguration(config))
-				return new Pair<Boolean, Long>(true, -2l);
+		if (!cfgManager.newConfiguration(config))
+			return new Pair<Boolean, Long>(true, -2l);
 
-			if (!prognosticator.prognosticate(config))
-				return new Pair<Boolean, Long>(true, -3l);
+		if (!prognosticator.prognosticate(config))
+			return new Pair<Boolean, Long>(true, -3l);
 
+		try {
 			if (!intermediateDraining())
 				return new Pair<Boolean, Long>(false, -4l);
 

From 9c3580745100e1bcb821240adbf7db74d72e39c5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Feb 2015 14:30:28 +0800
Subject: [PATCH 659/881] ProfileLogProcessor added.

---
 .../profiler/ProfileLogProcessor.java         | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
new file mode 100644
index 00000000..c227aa10
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
@@ -0,0 +1,97 @@
+package edu.mit.streamjit.impl.distributed.profiler;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Just an utility that processes profile.txt.
+ * 
+ * @author sumanan
+ * @since 4 Feb, 2015
+ */
+public class ProfileLogProcessor {
+
+	public static void main(String[] args) throws IOException {
+		String appName = "FilterBankPipeline";
+		// process1(appName);
+		process2(appName);
+	}
+
+	private static List<Integer> process1(String appName) throws IOException {
+		BufferedReader reader = new BufferedReader(new FileReader(
+				String.format("%s%sprofile.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedProfile.txt", appName, File.separator));
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("--------------------------------")) {
+				writer.write(line);
+				writer.write("\n");
+			}
+			if (line.contains("Not firable")) {
+				writer.write(line);
+				writer.write("\n");
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
+	private static List<Integer> process2(String appName) throws IOException {
+		Map<String, Integer> countMap = new HashMap<>();
+		BufferedReader reader = new BufferedReader(new FileReader(
+				String.format("%s%sprofile.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedProfile.txt", appName, File.separator));
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("--------------------------------")) {
+				writer.write(line);
+				writer.write("\n");
+				printStats(countMap);
+				System.out.println(line);
+			}
+			if (line.contains("Not firable")) {
+				String t = token(line);
+				if (!countMap.containsKey(t))
+					countMap.put(t, 0);
+				int val = countMap.get(t);
+				countMap.put(t, ++val);
+				writer.write(line);
+				writer.write("\n");
+			}
+		}
+		printStats(countMap);
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
+	private static void printStats(Map<String, Integer> countMap) {
+		for (Map.Entry<String, Integer> en : countMap.entrySet()) {
+			System.out.println(String.format("%s-%d", en.getKey(),
+					en.getValue()));
+		}
+		countMap.clear();
+	}
+
+	private static String token(String line) {
+		int start = line.indexOf('(');
+		int end = line.indexOf(')');
+		return String.format("Token%s", line.substring(start, end + 1));
+	}
+}

From 01e4e80f0487af486a2b381f328d1d2e0bd36611 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Feb 2015 14:32:13 +0800
Subject: [PATCH 660/881] Renamed: countMat->inputNotFirable.

---
 .../distributed/profiler/ProfileLogProcessor.java  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
index c227aa10..68cb1fa7 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
@@ -49,7 +49,7 @@ private static List<Integer> process1(String appName) throws IOException {
 	}
 
 	private static List<Integer> process2(String appName) throws IOException {
-		Map<String, Integer> countMap = new HashMap<>();
+		Map<String, Integer> inputNotFirable = new HashMap<>();
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sprofile.txt", appName, File.separator)));
 		FileWriter writer = new FileWriter(String.format(
@@ -61,20 +61,20 @@ private static List<Integer> process2(String appName) throws IOException {
 			if (line.startsWith("--------------------------------")) {
 				writer.write(line);
 				writer.write("\n");
-				printStats(countMap);
+				printStats(inputNotFirable);
 				System.out.println(line);
 			}
 			if (line.contains("Not firable")) {
 				String t = token(line);
-				if (!countMap.containsKey(t))
-					countMap.put(t, 0);
-				int val = countMap.get(t);
-				countMap.put(t, ++val);
+				if (!inputNotFirable.containsKey(t))
+					inputNotFirable.put(t, 0);
+				int val = inputNotFirable.get(t);
+				inputNotFirable.put(t, ++val);
 				writer.write(line);
 				writer.write("\n");
 			}
 		}
-		printStats(countMap);
+		printStats(inputNotFirable);
 		writer.flush();
 		reader.close();
 		writer.close();

From 79c52b90bf7f0b31dac41b282bb6eeba68a93569 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Feb 2015 14:51:26 +0800
Subject: [PATCH 661/881] Prints notFirable count of input&output separately

---
 .../profiler/ProfileLogProcessor.java         | 39 +++++++++++++------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
index 68cb1fa7..fcaca93c 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileLogProcessor.java
@@ -50,6 +50,8 @@ private static List<Integer> process1(String appName) throws IOException {
 
 	private static List<Integer> process2(String appName) throws IOException {
 		Map<String, Integer> inputNotFirable = new HashMap<>();
+		Map<String, Integer> outputNotFirable = new HashMap<>();
+		Map<String, Integer> notFirable = inputNotFirable;
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sprofile.txt", appName, File.separator)));
 		FileWriter writer = new FileWriter(String.format(
@@ -61,32 +63,47 @@ private static List<Integer> process2(String appName) throws IOException {
 			if (line.startsWith("--------------------------------")) {
 				writer.write(line);
 				writer.write("\n");
-				printStats(inputNotFirable);
+				printStats(inputNotFirable, outputNotFirable);
 				System.out.println(line);
-			}
+			} else if (line.contains("Input..."))
+				notFirable = inputNotFirable;
+			else if (line.contains("Output..."))
+				notFirable = outputNotFirable;
 			if (line.contains("Not firable")) {
 				String t = token(line);
-				if (!inputNotFirable.containsKey(t))
-					inputNotFirable.put(t, 0);
-				int val = inputNotFirable.get(t);
-				inputNotFirable.put(t, ++val);
+				add(notFirable, t);
 				writer.write(line);
 				writer.write("\n");
 			}
 		}
-		printStats(inputNotFirable);
+		printStats(inputNotFirable, outputNotFirable);
 		writer.flush();
 		reader.close();
 		writer.close();
 		return ret;
 	}
 
-	private static void printStats(Map<String, Integer> countMap) {
-		for (Map.Entry<String, Integer> en : countMap.entrySet()) {
-			System.out.println(String.format("%s-%d", en.getKey(),
+	private static void add(Map<String, Integer> notFirable, String t) {
+		if (!notFirable.containsKey(t))
+			notFirable.put(t, 0);
+		int val = notFirable.get(t);
+		notFirable.put(t, ++val);
+	}
+
+	private static void printStats(Map<String, Integer> inputNotFirable,
+			Map<String, Integer> outputNotFirable) {
+		System.out.println("Input...");
+		printStats(inputNotFirable);
+		System.out.println("Output...");
+		printStats(outputNotFirable);
+	}
+
+	private static void printStats(Map<String, Integer> notFirable) {
+		for (Map.Entry<String, Integer> en : notFirable.entrySet()) {
+			System.out.println(String.format("\t%s-%d", en.getKey(),
 					en.getValue()));
 		}
-		countMap.clear();
+		notFirable.clear();
 	}
 
 	private static String token(String line) {

From 7ccfdf4943dca48e1b0e6aa88922c0d3ff0ca923 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 00:08:48 +0800
Subject: [PATCH 662/881] Renamed:mainclass -> mainclasspath

---
 jarapp.properties | 4 ++--
 jarapp.xml        | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/jarapp.properties b/jarapp.properties
index b0b25183..5189550c 100644
--- a/jarapp.properties
+++ b/jarapp.properties
@@ -1,7 +1,7 @@
 #app=NestedSplitJoin
-#mainclass=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
+#mainclasspath=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
 app=FMRadio
-mainclass=edu.mit.streamjit.test.apps.fmradio.FMRadio
+mainclasspath=edu.mit.streamjit.test.apps.fmradio.FMRadio
 username=sumanan
 ##password=abc123
 
diff --git a/jarapp.xml b/jarapp.xml
index 694eb460..07e2624c 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -3,7 +3,7 @@
 <project default="exportAll" name="StreamJIT" basedir="/media/sumanan/D/StreamJit11/streamjit">
 	<property file="jarapp.properties" />
 	<!--property name="app" value="NestedSplitJoin" /-->
-	<!--property name="mainclass" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->
+	<!--property name="mainclasspath" value="edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin" /-->
 	<!-- If you want, you can hard code your username and the password as plain 
 		text here. Comment these properties (username or password or both) if you 
 		don't want to keep it as a plain text. If these properties are not set (commented), 
@@ -14,7 +14,7 @@
 	<target name="exportApp.jar" description="Exports ${app}.jar">
 		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
 			<manifest>
-				<attribute name="Main-Class" value="${mainclass}" />
+				<attribute name="Main-Class" value="${mainclasspath}" />
 				<attribute name="Class-Path" value="." />
 			</manifest>
 			<fileset dir="bin" />

From a42524e7b4a1b80493e898f635750260d5535f69 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 00:12:35 +0800
Subject: [PATCH 663/881] Ant:Sets mainclass from mainclasspath.

---
 jarapp.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index 07e2624c..5ad19c94 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -11,6 +11,11 @@
 	<!-- property name="username" value="sumanan" /-->
 	<!-- property name="password" value="" / -->
 
+	<script language="javascript">
+		arr = project.getProperty('mainclasspath').split('.');
+		project.setProperty('mainclass', arr[arr.length -1]);
+	 </script>
+
 	<target name="exportApp.jar" description="Exports ${app}.jar">
 		<jar destfile="${app}.jar" filesetmanifest="mergewithoutmain">
 			<manifest>

From 1280d72fbbeb5dc0b77b241aef2b99a094992fdc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 00:16:07 +0800
Subject: [PATCH 664/881] Ant:Remote setup target has been added.

Creates all appropriate directories. Makes soft links to 'data' and 'lib'.
Creates 'run.sh' and 'streamnode.sh' scripts.
---
 jarapp.properties | 1 +
 jarapp.xml        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/jarapp.properties b/jarapp.properties
index 5189550c..c927f734 100644
--- a/jarapp.properties
+++ b/jarapp.properties
@@ -4,4 +4,5 @@ app=FMRadio
 mainclasspath=edu.mit.streamjit.test.apps.fmradio.FMRadio
 username=sumanan
 ##password=abc123
+numOfSNs=2
 
diff --git a/jarapp.xml b/jarapp.xml
index 5ad19c94..d4cc8972 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -100,7 +100,7 @@
 	</target>
 
 	<target name="remotesetup" depends="getLogin">
-		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/helloworld.sh" input="kkkk" />
+		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${mainclass} ${numOfSNs}" />
 	</target>
 
 </project>

From d6cc1e9a670223fa3abc6cffefd0466fdcd2307d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 01:24:16 +0800
Subject: [PATCH 665/881] Ant:target "remotesetup" depends on "sendtoLanka"

---
 jarapp.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index d4cc8972..27eaa955 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -99,7 +99,7 @@
 		<!--echo message="Password: ${password}" /-->
 	</target>
 
-	<target name="remotesetup" depends="getLogin">
+	<target name="remotesetup" depends="sendtoLanka">
 		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${mainclass} ${numOfSNs}" />
 	</target>
 

From b75ce85e7593e25001d3d168b078c1c0b3a668ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 11:04:01 +0800
Subject: [PATCH 666/881] TimeLogProcessor:Handles negative running time

OnlineTuner.java may report negative running time if the new confg is invalid
or the reconfiguration failed due to some other reasons. TimeLogProcessor
ignores negative values when calculating min running time.
---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index dc4745ad..1de24402 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -81,6 +81,8 @@ private static List<Integer> processRunTime(String appName)
 				String time = arr[3].trim();
 				time = time.substring(0, time.length() - 2);
 				int val = Integer.parseInt(time);
+				if (val < 1)
+					val = 2 * min;
 				min = Math.min(min, val);
 				ret.add(val);
 				String data = String

From f32be9e1a4e235fedf427a935582e0389a670374 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 11:36:44 +0800
Subject: [PATCH 667/881] TimeLogProcessor:Processing heapStatus is stopped.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 1de24402..b32e4c89 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -35,8 +35,8 @@ public static void main(String[] args) throws IOException {
 		}
 		writer.close();
 
-		writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
-		writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
+		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
+		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
 
 	private static List<Integer> processCompileTime(String appName)

From ddf9be13d4c070bd44e4d2879529aa1e4e139c89 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 11:46:14 +0800
Subject: [PATCH 668/881] TimeLogProcessor:Method summarize() has been added

Codes in the main() method have been moved to the new method summariza().
---
 .../mit/streamjit/util/TimeLogProcessor.java  | 44 ++++++++++---------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index b32e4c89..a168c271 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -17,26 +17,7 @@
 public class TimeLogProcessor {
 
 	public static void main(String[] args) throws IOException {
-		String appName = "FMRadioCore";
-		List<Integer> compileTime = processCompileTime(appName);
-		List<Integer> runTime = processRunTime(appName);
-		List<Integer> drainTime = processDrainTime(appName);
-
-		FileWriter writer = new FileWriter(String.format("%s%stotalStats.txt",
-				appName, File.separator));
-		int min = Integer.MAX_VALUE;
-
-		for (int i = 0; i < runTime.size(); i++) {
-			min = Math.min(min, runTime.get(i));
-			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n", i + 1,
-					compileTime.get(i), runTime.get(i), drainTime.get(i), min);
-
-			writer.write(msg);
-		}
-		writer.close();
-
-		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
-		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
+		summarize("FMRadioCore");
 	}
 
 	private static List<Integer> processCompileTime(String appName)
@@ -64,6 +45,7 @@ private static List<Integer> processCompileTime(String appName)
 		writer.close();
 		return ret;
 	}
+
 	private static List<Integer> processRunTime(String appName)
 			throws IOException {
 
@@ -156,4 +138,26 @@ private static List<Integer> processSNHeap(String fileName,
 		reader.close();
 		return ret;
 	}
+
+	private static void summarize(String appName) throws IOException {
+		List<Integer> compileTime = processCompileTime(appName);
+		List<Integer> runTime = processRunTime(appName);
+		List<Integer> drainTime = processDrainTime(appName);
+
+		FileWriter writer = new FileWriter(String.format("%s%stotalStats.txt",
+				appName, File.separator));
+		int min = Integer.MAX_VALUE;
+
+		for (int i = 0; i < runTime.size(); i++) {
+			min = Math.min(min, runTime.get(i));
+			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n", i + 1,
+					compileTime.get(i), runTime.get(i), drainTime.get(i), min);
+
+			writer.write(msg);
+		}
+		writer.close();
+
+		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
+		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
+	}
 }

From 67dc59d7524263f6573562a3b26be442845eb5be Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 11:53:05 +0800
Subject: [PATCH 669/881] Utils.createDir() has been made public.

---
 src/edu/mit/streamjit/impl/distributed/common/Utils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index b83c2014..c4365e54 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -84,7 +84,7 @@ public static void printOutOfMemory() {
 	 * @return <code>true</code> if and only if the directory was created; false
 	 *         otherwise.
 	 */
-	private static boolean createDir(String name) {
+	public static boolean createDir(String name) {
 		File dir = new File(name);
 		if (dir.exists()) {
 			if (dir.isDirectory())

From be0a2d0437406446066a566e1988bc87c709482b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 11:59:47 +0800
Subject: [PATCH 670/881] TimeLogProcessor:Creates Summary Dir

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index a168c271..2929b3f1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -8,6 +8,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import edu.mit.streamjit.impl.distributed.common.Utils;
+
 /**
  * Processes the Distributed StreamJit's time log files and generate summary.
  * 
@@ -144,8 +146,12 @@ private static void summarize(String appName) throws IOException {
 		List<Integer> runTime = processRunTime(appName);
 		List<Integer> drainTime = processDrainTime(appName);
 
+		String summaryDir = String.format("%s%ssummary", appName,
+				File.separator);
+		Utils.createDir(summaryDir);
+
 		FileWriter writer = new FileWriter(String.format("%s%stotalStats.txt",
-				appName, File.separator));
+				summaryDir, File.separator));
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 0; i < runTime.size(); i++) {

From 1c2a64c2bebf3faf9a7a7b8eb04f19fa8cb5352c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:03:28 +0800
Subject: [PATCH 671/881] TimeLogProcessor:makePlotFile() has been added.

---
 .../mit/streamjit/util/TimeLogProcessor.java    | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 2929b3f1..c2b78688 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -166,4 +166,21 @@ private static void summarize(String appName) throws IOException {
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
+
+	private static void makePlotFile(File dir, String name, String dataFile)
+			throws IOException {
+		File plotfile = new File(dir, "plot.plt");
+		FileWriter writer = new FileWriter(plotfile, false);
+		writer.write("set terminal postscript eps enhanced color\n");
+		writer.write(String.format("set output \"%s.eps\"\n", name));
+		writer.write("set ylabel \"Time(ms)\"\n");
+		writer.write("set xlabel \"Tuning Rounds\"\n");
+		writer.write(String.format("set title \"%s\"\n", name));
+		writer.write("set grid\n");
+		writer.write("#set yrange [0:*]\n");
+		writer.write(String
+				.format("plot \"%s\" using 1:3 with linespoints title \"Current best running time\"",
+						dataFile));
+		writer.close();
+	}
 }

From b2bfcdc3c04a297a4fb30160871f4bc4200726ed Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:13:24 +0800
Subject: [PATCH 672/881] TimeLogProcessor:Plots compile, Run & Drain times

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index c2b78688..79bfa717 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -163,6 +163,8 @@ private static void summarize(String appName) throws IOException {
 		}
 		writer.close();
 
+		makePlotFile(new File(summaryDir), appName, "totalStats.txt");
+
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
@@ -179,7 +181,16 @@ private static void makePlotFile(File dir, String name, String dataFile)
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
-				.format("plot \"%s\" using 1:3 with linespoints title \"Current best running time\"",
+				.format("plot \"%s\" using 1:5 with linespoints title \"Current best running time\"\n",
+						dataFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:2 with linespoints title \"Compile time\"\n",
+						dataFile));
+		writer.write(String.format(
+				"plot \"%s\" using 1:3 with linespoints title \"Run time\"\n",
+				dataFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:4 with linespoints title \"Drain time\"\n",
 						dataFile));
 		writer.close();
 	}

From 0d9700b38922792c21f5d576151f892e7d766329 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:16:09 +0800
Subject: [PATCH 673/881] TimeLogProcessor:local variable dataFile added.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 79bfa717..29d4550d 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -145,13 +145,14 @@ private static void summarize(String appName) throws IOException {
 		List<Integer> compileTime = processCompileTime(appName);
 		List<Integer> runTime = processRunTime(appName);
 		List<Integer> drainTime = processDrainTime(appName);
+		String dataFile = "totalStats.txt";
 
 		String summaryDir = String.format("%s%ssummary", appName,
 				File.separator);
 		Utils.createDir(summaryDir);
 
-		FileWriter writer = new FileWriter(String.format("%s%stotalStats.txt",
-				summaryDir, File.separator));
+		FileWriter writer = new FileWriter(String.format("%s%s%s", summaryDir,
+				File.separator, dataFile));
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 0; i < runTime.size(); i++) {
@@ -163,7 +164,7 @@ private static void summarize(String appName) throws IOException {
 		}
 		writer.close();
 
-		makePlotFile(new File(summaryDir), appName, "totalStats.txt");
+		makePlotFile(new File(summaryDir), appName, dataFile);
 
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));

From 0aefc3150d6dfee8c6256540b79a73c3dc578523 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:22:19 +0800
Subject: [PATCH 674/881] TimeLogProcessor: plot() added

Calls "/usr/bin/gnuplot" with "plot.plt" to generate the graphs.
---
 .../mit/streamjit/util/TimeLogProcessor.java  | 36 ++++++++++++++++---
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 29d4550d..583bad9e 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -5,6 +5,8 @@
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -147,9 +149,11 @@ private static void summarize(String appName) throws IOException {
 		List<Integer> drainTime = processDrainTime(appName);
 		String dataFile = "totalStats.txt";
 
-		String summaryDir = String.format("%s%ssummary", appName,
-				File.separator);
-		Utils.createDir(summaryDir);
+		// String summaryDir = String.format("%s%ssummary", appName,
+		// File.separator);
+		File summaryDir = new File(String.format("%s%ssummary", appName,
+				File.separator));
+		Utils.createDir(summaryDir.getPath());
 
 		FileWriter writer = new FileWriter(String.format("%s%s%s", summaryDir,
 				File.separator, dataFile));
@@ -164,7 +168,8 @@ private static void summarize(String appName) throws IOException {
 		}
 		writer.close();
 
-		makePlotFile(new File(summaryDir), appName, dataFile);
+		makePlotFile(summaryDir, appName, dataFile);
+		plot(summaryDir);
 
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
@@ -195,4 +200,27 @@ private static void makePlotFile(File dir, String name, String dataFile)
 						dataFile));
 		writer.close();
 	}
+
+	private static void plot(File dir) throws IOException {
+		String[] s = { "/usr/bin/gnuplot", "plot.plt" };
+		try {
+			ProcessBuilder pb = new ProcessBuilder(s);
+			pb.directory(dir);
+			Process proc = pb.start();
+			InputStream stdin = (InputStream) proc.getErrorStream();
+			InputStreamReader isr = new InputStreamReader(stdin);
+			BufferedReader br = new BufferedReader(isr);
+			String line = null;
+			while ((line = br.readLine()) != null)
+				System.err.println("gnuplot:" + line);
+			int exitVal = proc.waitFor();
+			if (exitVal != 0)
+				System.out.println("gnuplot Process exitValue: " + exitVal);
+			proc.getInputStream().close();
+			proc.getOutputStream().close();
+			proc.getErrorStream().close();
+		} catch (Exception e) {
+			System.err.println("Fail: " + e);
+		}
+	}
 }

From 01711537c8fe69455aaab21c245116e0a437cc53 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:29:31 +0800
Subject: [PATCH 675/881] TimeLogProcessor:summarize() has been made public.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 583bad9e..a96f74a6 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -143,7 +143,7 @@ private static List<Integer> processSNHeap(String fileName,
 		return ret;
 	}
 
-	private static void summarize(String appName) throws IOException {
+	public static void summarize(String appName) throws IOException {
 		List<Integer> compileTime = processCompileTime(appName);
 		List<Integer> runTime = processRunTime(appName);
 		List<Integer> drainTime = processDrainTime(appName);

From 7b8a3af4101b44fde71599c015669b47074a4f51 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 12:31:12 +0800
Subject: [PATCH 676/881] OnlineTuner calls TimeLogProcessor.Summarize()

At the end of the tuning, Online tuner summarize the running time and plot
graphs.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 71bc2dc1..f0aef929 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -18,6 +18,7 @@
 import edu.mit.streamjit.tuner.TCPTuner;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
+import edu.mit.streamjit.util.TimeLogProcessor;
 import edu.mit.streamjit.util.json.Jsonifiers;
 
 /**
@@ -123,6 +124,12 @@ private void tune() {
 
 		if (needTermination)
 			terminate();
+
+		try {
+			TimeLogProcessor.summarize(app.name);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
 	}
 
 	private void startTuner() throws IOException {

From b3b1d39dd6ad78b88ebaac0d9d3a4b3187602c8a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 13:11:42 +0800
Subject: [PATCH 677/881] Uses BlockingTailChannel2.

---
 src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 18e67826..c268c3d1 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -344,7 +344,7 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
-		tailChannel = new TailChannels.BlockingTailChannel1(
+		tailChannel = new TailChannels.BlockingTailChannel2(
 				bufferMap.get(tailToken), controller.getConProvider(),
 				tailconInfo, "tailChannel - " + tailToken.toString(), 0,
 				skipCount, GlobalConstants.outputCount, app.name);

From 382006b89d989dc4dee2b74006fd18e1e7be7c94 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 23:43:09 +0800
Subject: [PATCH 678/881] TCPTuner:Passes relative path instead of abs path.

TCPTuner.startTuner() takes streamjit.py's relative path as the first arg.
---
 src/edu/mit/streamjit/tuner/TCPTuner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index 008f3313..a81588a8 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -189,7 +189,7 @@ public static void main(String[] args) throws InterruptedException,
 
 		OpenTuner tuner = new TCPTuner();
 		try {
-			tuner.startTuner("/lib/opentuner/streamjit/streamjit.py", new File(
+			tuner.startTuner("./lib/opentuner/streamjit/streamjit.py", new File(
 					System.getProperty("user.dir")));
 		} catch (IOException e) {
 			e.printStackTrace();

From 213575cf7bac7ab8e9b7f7c5605419f64b5f7b18 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Feb 2015 23:51:26 +0800
Subject: [PATCH 679/881] Ant:New property 'streamGraphName' has been added

We need the app's oneToOneElement's name to create app directory.
run.sh creates appDir before running app.jar.
---
 jarapp.properties | 2 ++
 jarapp.xml        | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/jarapp.properties b/jarapp.properties
index c927f734..ab081c55 100644
--- a/jarapp.properties
+++ b/jarapp.properties
@@ -1,7 +1,9 @@
 #app=NestedSplitJoin
 #mainclasspath=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
+#streamGraphName=NestedSplitJoinCore
 app=FMRadio
 mainclasspath=edu.mit.streamjit.test.apps.fmradio.FMRadio
+streamGraphName=FMRadioCore
 username=sumanan
 ##password=abc123
 numOfSNs=2
diff --git a/jarapp.xml b/jarapp.xml
index 27eaa955..87f96aa4 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -100,7 +100,7 @@
 	</target>
 
 	<target name="remotesetup" depends="sendtoLanka">
-		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${mainclass} ${numOfSNs}" />
+		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${streamGraphName} ${numOfSNs}" />
 	</target>
 
 </project>

From 64cf86b0cabeea5be2aab60b35b6eabcda9de3f2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Feb 2015 10:36:38 +0800
Subject: [PATCH 680/881] Ant: compile target has been added.

Compile the project using Ant script has some issues. Need to solve soon.
---
 jarapp.xml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index 87f96aa4..40f1698b 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -103,4 +103,31 @@
 		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${streamGraphName} ${numOfSNs}" />
 	</target>
 
+	<!--http://www.vogella.com/tutorials/ApacheAnt/article.html-->
+	<!-- The value of a property is accessed via ${} -->
+	<property name="src.dir" location="src" />
+	<property name="lib.dir" location="lib" />
+	<property name="build.dir" location="bin" />
+
+	<!--Create a classpath container which can be later used in the ant task-->
+	<path id="build.classpath">
+		<fileset dir="${lib.dir}">
+			<include name="**/*.jar" />
+		</fileset>
+	</path>
+
+	<!-- Deletes the existing build directory-->
+	<target name="clean">
+		<delete dir="${build.dir}" />
+	</target>
+
+	<!-- Creates the  build  directory-->
+	<target name="makedir">
+		<mkdir dir="${build.dir}" />
+	</target>
+
+	<!-- Compiles the java code -->
+	<target name="compile" depends="clean, makedir">
+		<javac srcdir="${src.dir}" destdir="${build.dir}" classpathref="build.classpath" />
+	</target>
 </project>

From 7d166beee4accfc376091271c1efdd5c45f4adf7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 20:30:06 +0800
Subject: [PATCH 681/881] Spelling correction: draing->draining.

---
 src/edu/mit/streamjit/impl/common/AbstractDrainer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
index ff50249a..668216b7 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
@@ -210,7 +210,7 @@ public final boolean startDraining(int type) {
 		} else if (state == DrainerState.FINAL) {
 			return false;
 		} else {
-			throw new RuntimeException("Drainer is in draing mode.");
+			throw new RuntimeException("Drainer is in draining mode.");
 		}
 	}
 

From 86addc963f4a8f508bd4de2a236147ef9673f072 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 20:57:38 +0800
Subject: [PATCH 682/881] Renamed:ConfigurationManager->PartitionManager

---
 .../mit/streamjit/impl/common/ConfigurationEditor.java    | 8 ++++----
 .../impl/distributed/DistributedBlobFactory.java          | 6 +++---
 .../impl/distributed/DistributedStreamCompiler.java       | 4 ++--
 src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java | 2 +-
 .../{ConfigurationManager.java => PartitionManager.java}  | 4 ++--
 .../streamjit/impl/distributed/StreamJitAppManager.java   | 4 ++--
 src/edu/mit/streamjit/impl/distributed/Visualizer.java    | 2 +-
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java | 2 +-
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java  | 6 +++---
 9 files changed, 19 insertions(+), 19 deletions(-)
 rename src/edu/mit/streamjit/impl/distributed/{ConfigurationManager.java => PartitionManager.java} (99%)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index 9e75bf37..eddd6dae 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,7 +16,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
-import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager;
 import edu.mit.streamjit.impl.distributed.ConnectionManager;
 import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
@@ -83,7 +83,7 @@ private static Pair<String, Integer> generate(OneToOneElement<?, ?> stream,
 
 	/**
 	 * This edit is for the configurations which are generated using
-	 * {@link WorkerMachine} as {@link ConfigurationManager}.
+	 * {@link WorkerMachine} as {@link PartitionManager}.
 	 */
 	private static void edit(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
@@ -116,7 +116,7 @@ private static Pair<String, Integer> generate1(
 			OneToOneElement<?, ?> stream, int noOfnodes) {
 		StreamJitApp<?, ?> app = new StreamJitApp<>(stream);
 		int noofwrks = Workers.getIdentifier(app.sink) + 1;
-		ConfigurationManager cfgManager = new HotSpotTuning(app);
+		PartitionManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManger = new BlockingTCPNoParams(0);
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManger,
 				noOfnodes);
@@ -152,7 +152,7 @@ private static void generate2(OneToOneElement<?, ?> stream) {
 
 	/**
 	 * This edit is for the configurations which are generated using
-	 * {@link HotSpotTuning} as {@link ConfigurationManager}.
+	 * {@link HotSpotTuning} as {@link PartitionManager}.
 	 */
 	private static void edit1(String appName, String namePrefix, int maxWor)
 			throws NumberFormatException, IOException {
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 86618bff..25ee1e17 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -35,11 +35,11 @@ public class DistributedBlobFactory implements BlobFactory {
 
 	private int noOfMachines;
 
-	private final ConfigurationManager cfgManager;
+	private final PartitionManager cfgManager;
 
 	private final ConnectionManager connectionManager;
 
-	public DistributedBlobFactory(ConfigurationManager cfgManager,
+	public DistributedBlobFactory(PartitionManager cfgManager,
 			ConnectionManager connectionManager, int noOfMachines) {
 		this.cfgManager = cfgManager;
 		this.noOfMachines = noOfMachines;
@@ -47,7 +47,7 @@ public DistributedBlobFactory(ConfigurationManager cfgManager,
 	}
 
 	/**
-	 * If {@link ConfigurationManager} is not passed as a constructor argument
+	 * If {@link PartitionManager} is not passed as a constructor argument
 	 * then {@link WorkerMachine} will be used as default one.
 	 * 
 	 * @param noOfMachines
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 8ae76c9d..7274f735 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -117,7 +117,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		StreamJitApp<I, O> app = new StreamJitApp<>(stream);
 		Controller controller = establishController();
 
-		ConfigurationManager cfgManager = new HotSpotTuning(app);
+		PartitionManager cfgManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
 		setConfiguration(controller, app, cfgManager, conManager);
@@ -252,7 +252,7 @@ public void drain() {
 	}
 
 	private <I, O> void setConfiguration(Controller controller,
-			StreamJitApp<I, O> app, ConfigurationManager cfgManager,
+			StreamJitApp<I, O> app, PartitionManager cfgManager,
 			ConnectionManager conManager) {
 		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
 				Math.max(noOfnodes - 1, 1));
diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 537d4d1b..ae73410f 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -21,7 +21,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.ConfigurationManager.AbstractConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractConfigurationManager;
 
 public final class HotSpotTuning extends AbstractConfigurationManager {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
similarity index 99%
rename from src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
rename to src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index 44ca905f..35c9c842 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -47,7 +47,7 @@
  * @since Jan 16, 2014
  * 
  */
-public interface ConfigurationManager {
+public interface PartitionManager {
 
 	/**
 	 * Generates default configuration with all tuning parameters for tuning.
@@ -98,7 +98,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 * @since Jan 17, 2014
 	 */
 	public static abstract class AbstractConfigurationManager implements
-			ConfigurationManager {
+			PartitionManager {
 
 		protected final StreamJitApp<?, ?> app;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index c268c3d1..754d7132 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -54,7 +54,7 @@ public class StreamJitAppManager {
 
 	private AppStatusProcessorImpl apStsPro = null;
 
-	private final ConfigurationManager cfgManager;
+	private final PartitionManager cfgManager;
 
 	private Map<Token, ConnectionInfo> conInfoMap;
 
@@ -108,7 +108,7 @@ public class StreamJitAppManager {
 	private final SNTimeInfoProcessor timeInfoProcessor;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
-			ConfigurationManager cfgManager, ConnectionManager conManager,
+			PartitionManager cfgManager, ConnectionManager conManager,
 			TimeLogger logger) {
 		this.controller = controller;
 		this.app = app;
diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 86b00287..8862f9f0 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -43,7 +43,7 @@ public interface Visualizer {
 
 	/**
 	 * Partitions Machine Map of the current configuration. Only the
-	 * {@link ConfigurationManager} has the information to generate this map.
+	 * {@link PartitionManager} has the information to generate this map.
 	 * Visualizer has no glue to generate this partitionsMachineMap.
 	 * 
 	 * @param partitionsMachineMap
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 15c0144d..1047d966 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.ConfigurationManager.AbstractConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractConfigurationManager;
 
 /**
  * This class implements one type of search space. Adds "worker to machine"
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f0aef929..e38d815d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.TimeLogger;
-import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
@@ -32,13 +32,13 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitAppManager manager;
 	private final OpenTuner tuner;
 	private final StreamJitApp<?, ?> app;
-	private final ConfigurationManager cfgManager;
+	private final PartitionManager cfgManager;
 	private final boolean needTermination;
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
-			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
+			StreamJitApp<?, ?> app, PartitionManager cfgManager,
 			TimeLogger logger, boolean needTermination) {
 		this.drainer = drainer;
 		this.manager = manager;

From d3d5821f9288393347ddec616ef01ea825d83e4f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 21:00:10 +0800
Subject: [PATCH 683/881] 
 AbstractConfigurationManager->AbstractPartitionManager

---
 src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java    | 4 ++--
 src/edu/mit/streamjit/impl/distributed/PartitionManager.java | 4 ++--
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index ae73410f..ad72bee4 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -21,9 +21,9 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractPartitionManager;
 
-public final class HotSpotTuning extends AbstractConfigurationManager {
+public final class HotSpotTuning extends AbstractPartitionManager {
 
 	Map<Integer, List<Worker<?, ?>>> partitionGroup;
 	Map<Splitter<?, ?>, Set<Worker<?, ?>>> skippedSplitters;
diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index 35c9c842..de08af70 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -97,12 +97,12 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 * @author Sumanan sumanan@mit.edu
 	 * @since Jan 17, 2014
 	 */
-	public static abstract class AbstractConfigurationManager implements
+	public static abstract class AbstractPartitionManager implements
 			PartitionManager {
 
 		protected final StreamJitApp<?, ?> app;
 
-		AbstractConfigurationManager(StreamJitApp<?, ?> app) {
+		AbstractPartitionManager(StreamJitApp<?, ?> app) {
 			this.app = app;
 		}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 1047d966..3899f678 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -17,7 +17,7 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager.AbstractPartitionManager;
 
 /**
  * This class implements one type of search space. Adds "worker to machine"
@@ -37,7 +37,7 @@
  * @since Jan 16, 2014
  * 
  */
-public final class WorkerMachine extends AbstractConfigurationManager {
+public final class WorkerMachine extends AbstractPartitionManager {
 
 	WorkerMachine(StreamJitApp app) {
 		super(app);

From 6544972f901d33100a42751702eae394536a8acc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 21:15:09 +0800
Subject: [PATCH 684/881] PartitionManager's comments modified.

PartitionManager's comments modified to reflect latest changes.
---
 .../impl/distributed/PartitionManager.java         | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index de08af70..e08f9a9f 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -31,18 +31,16 @@
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
 /**
- * ConfigurationManager deals with {@link Configuration}. Mainly, It does
- * following two tasks.
+ * PartitionManager is responsible to partition a stream graph for a cluster.
+ * Partitioning process can be tuned. Implementations of this interface must
+ * provide the following two tasks.
  * <ol>
- * <li>Generates configuration with appropriate tuning parameters (Based on the
+ * <li>Generate configuration with appropriate tuning parameters (Based on the
  * search space design strategy) for tuning.
- * <li>Dispatch the configuration given by the open tuner and make blobs
- * accordingly.
+ * <li>Dispatch a new configuration given by the open tuner and generate
+ * partition machine map.
  * </ol>
  * 
- * One can implement this interface to try different search space designs as he
- * want.
- * 
  * @author Sumanan sumanan@mit.edu
  * @since Jan 16, 2014
  * 

From 293cc77cc18386cab2d9f486b452c43b9bb152f3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 22:10:50 +0800
Subject: [PATCH 685/881] getStaticCfg() has been moved to StreamJitapp.

PartitionManager.getStaticConfiguration() has been moved to StreamJitApp class.
---
 .../impl/distributed/PartitionManager.java     | 18 ------------------
 .../impl/distributed/StreamJitApp.java         | 16 ++++++++++++++++
 .../impl/distributed/StreamJitAppManager.java  |  6 +++---
 3 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index e08f9a9f..2343cf05 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -70,15 +70,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 */
 	public boolean newConfiguration(Configuration config);
 
-	/**
-	 * Generates static information of the app that is needed by steramnodes.
-	 * This configuration will be sent to streamnodes when setting up a new app
-	 * for execution (Only once).
-	 * 
-	 * @return static information of the app that is needed by steramnodes.
-	 */
-	public Configuration getStaticConfiguration();
-
 	/**
 	 * For every reconfiguration, this method may be called by the appropriate
 	 * class to get new configuration information that can be sent to all
@@ -104,15 +95,6 @@ public static abstract class AbstractPartitionManager implements
 			this.app = app;
 		}
 
-		@Override
-		public Configuration getStaticConfiguration() {
-			Configuration.Builder builder = Configuration.builder();
-			builder.putExtraData(GlobalConstants.JARFILE_PATH, app.jarFilePath)
-					.putExtraData(GlobalConstants.TOPLEVEL_WORKER_NAME,
-							app.topLevelClass);
-			return builder.build();
-		}
-
 		@Override
 		public Configuration getDynamicConfiguration() {
 			Configuration.Builder builder = Configuration.builder();
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index b063234a..55e20460 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -31,6 +31,7 @@
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.impl.interp.Interpreter;
@@ -341,4 +342,19 @@ private List<MessageConstraint> getConstrains() {
 	public Set<List<Integer>> paths() {
 		return StreamPathBuilder.paths(streamGraph);
 	}
+
+	/**
+	 * Static information of the {@link StreamJitApp} that is essential for
+	 * {@link StreamNode}s to set up. This configuration will be sent to
+	 * {@link StreamNode}s when setting up a new app (Only once).
+	 * 
+	 * @return static information of the app that is needed by steramnodes.
+	 */
+	public Configuration getStaticConfiguration() {
+		Configuration.Builder builder = Configuration.builder();
+		builder.putExtraData(GlobalConstants.JARFILE_PATH, jarFilePath)
+				.putExtraData(GlobalConstants.TOPLEVEL_WORKER_NAME,
+						topLevelClass);
+		return builder.build();
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 754d7132..b1cd19ae 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -122,9 +122,9 @@ public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
 		this.apStsPro = new AppStatusProcessorImpl(controller.getAllNodeIDs()
 				.size());
 		controller.registerManager(this);
-		controller.newApp(cfgManager.getStaticConfiguration()); // TODO: Find a
-																// good calling
-																// place.
+		controller.newApp(app.getStaticConfiguration()); // TODO: Find a
+															// good calling
+															// place.
 		isRunning = false;
 
 		headToken = Token.createOverallInputToken(app.source);

From b1c9fcd54a0cddb0673c60c84143fe27760ead15 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 13 Feb 2015 22:17:40 +0800
Subject: [PATCH 686/881] getDynamicCfg() has been moved to StreamJitApp

PartitionManager.getDynamicConfiguration() has been moved to StreamJitApp class.
---
 .../impl/distributed/PartitionManager.java    | 78 -------------------
 .../impl/distributed/StreamJitApp.java        | 69 ++++++++++++++++
 .../impl/distributed/StreamJitAppManager.java |  3 +-
 3 files changed, 70 insertions(+), 80 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index 2343cf05..5ced26d8 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -2,7 +2,6 @@
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -15,19 +14,8 @@
 import edu.mit.streamjit.api.Joiner;
 import edu.mit.streamjit.api.Splitter;
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
-import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
-import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
-import edu.mit.streamjit.impl.distributed.common.Utils;
-import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.interp.ChannelFactory;
-import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.AbstractPartitioner;
 
 /**
@@ -70,15 +58,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 */
 	public boolean newConfiguration(Configuration config);
 
-	/**
-	 * For every reconfiguration, this method may be called by the appropriate
-	 * class to get new configuration information that can be sent to all
-	 * participating {@link StreamNode}s.
-	 * 
-	 * @return new partition information
-	 */
-	public Configuration getDynamicConfiguration();
-
 	/**
 	 * Implements the functions those can be called by runtimer to send
 	 * configuration information to streamnodes.
@@ -95,63 +74,6 @@ public static abstract class AbstractPartitionManager implements
 			this.app = app;
 		}
 
-		@Override
-		public Configuration getDynamicConfiguration() {
-			Configuration.Builder builder = Configuration.builder();
-
-			Map<Integer, Integer> coresPerMachine = new HashMap<>();
-			for (Entry<Integer, List<Set<Worker<?, ?>>>> machine : app.partitionsMachineMap
-					.entrySet()) {
-				coresPerMachine
-						.put(machine.getKey(), machine.getValue().size());
-			}
-
-			PartitionParameter.Builder partParam = PartitionParameter.builder(
-					GlobalConstants.PARTITION, coresPerMachine);
-
-			BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
-			BlobFactory comp2Factory = new Compiler2BlobFactory();
-			partParam.addBlobFactory(intFactory);
-			partParam.addBlobFactory(comp2Factory);
-			app.blobtoMachineMap = new HashMap<>();
-
-			BlobFactory bf = GlobalConstants.useCompilerBlob ? comp2Factory
-					: intFactory;
-			for (Integer machineID : app.partitionsMachineMap.keySet()) {
-				List<Set<Worker<?, ?>>> blobList = app.partitionsMachineMap
-						.get(machineID);
-				for (Set<Worker<?, ?>> blobWorkers : blobList) {
-					// TODO: One core per blob. Need to change this.
-					partParam.addBlob(machineID, 1, bf, blobWorkers);
-
-					// TODO: Temp fix to build.
-					Token t = Utils.getblobID(blobWorkers);
-					app.blobtoMachineMap.put(t, machineID);
-				}
-			}
-
-			builder.addParameter(partParam.build());
-			if (GlobalConstants.useCompilerBlob)
-				builder.addSubconfiguration("blobConfigs",
-						app.getConfiguration());
-			else
-				builder.addSubconfiguration("blobConfigs",
-						getInterpreterConfg());
-			return builder.build();
-		}
-
-		private Configuration getInterpreterConfg() {
-			Configuration.Builder builder = Configuration.builder();
-			List<ChannelFactory> universe = Arrays
-					.<ChannelFactory> asList(new ConcurrentChannelFactory());
-			SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
-					"channelFactory", ChannelFactory.class, universe.get(0),
-					universe);
-
-			builder.addParameter(cfParameter);
-			return builder.build();
-		}
-
 		/**
 		 * Copied form {@link AbstractPartitioner} class. But modified to
 		 * support nested splitjoiners.</p> Returns all {@link Worker}s in a
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 55e20460..85084cf6 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -1,9 +1,12 @@
 package edu.mit.streamjit.impl.distributed;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -24,16 +27,21 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
+import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
 import edu.mit.streamjit.impl.common.MessageConstraint;
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.util.Pair;
 
@@ -357,4 +365,65 @@ public Configuration getStaticConfiguration() {
 						topLevelClass);
 		return builder.build();
 	}
+
+	/**
+	 * For every reconfiguration, this method may be called by an appropriate
+	 * class to get new configuration information that can be sent to all
+	 * participating {@link StreamNode}s. Mainly this configuration contains
+	 * partition information.
+	 * 
+	 * @return new partition information
+	 */
+	public Configuration getDynamicConfiguration() {
+		Configuration.Builder builder = Configuration.builder();
+
+		Map<Integer, Integer> coresPerMachine = new HashMap<>();
+		for (Entry<Integer, List<Set<Worker<?, ?>>>> machine : partitionsMachineMap
+				.entrySet()) {
+			coresPerMachine.put(machine.getKey(), machine.getValue().size());
+		}
+
+		PartitionParameter.Builder partParam = PartitionParameter.builder(
+				GlobalConstants.PARTITION, coresPerMachine);
+
+		BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
+		BlobFactory comp2Factory = new Compiler2BlobFactory();
+		partParam.addBlobFactory(intFactory);
+		partParam.addBlobFactory(comp2Factory);
+		blobtoMachineMap = new HashMap<>();
+
+		BlobFactory bf = GlobalConstants.useCompilerBlob ? comp2Factory
+				: intFactory;
+		for (Integer machineID : partitionsMachineMap.keySet()) {
+			List<Set<Worker<?, ?>>> blobList = partitionsMachineMap
+					.get(machineID);
+			for (Set<Worker<?, ?>> blobWorkers : blobList) {
+				// TODO: One core per blob. Need to change this.
+				partParam.addBlob(machineID, 1, bf, blobWorkers);
+
+				// TODO: Temp fix to build.
+				Token t = Utils.getblobID(blobWorkers);
+				blobtoMachineMap.put(t, machineID);
+			}
+		}
+
+		builder.addParameter(partParam.build());
+		if (GlobalConstants.useCompilerBlob)
+			builder.addSubconfiguration("blobConfigs", getConfiguration());
+		else
+			builder.addSubconfiguration("blobConfigs", getInterpreterConfg());
+		return builder.build();
+	}
+
+	private Configuration getInterpreterConfg() {
+		Configuration.Builder builder = Configuration.builder();
+		List<ChannelFactory> universe = Arrays
+				.<ChannelFactory> asList(new ConcurrentChannelFactory());
+		SwitchParameter<ChannelFactory> cfParameter = new SwitchParameter<ChannelFactory>(
+				"channelFactory", ChannelFactory.class, universe.get(0),
+				universe);
+
+		builder.addParameter(cfParameter);
+		return builder.build();
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index b1cd19ae..af3b6f01 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -234,7 +234,7 @@ public boolean isRunning() {
 
 	public boolean reconfigure(int multiplier) {
 		reset();
-		Configuration.Builder builder = Configuration.builder(cfgManager
+		Configuration.Builder builder = Configuration.builder(app
 				.getDynamicConfiguration());
 
 		conInfoMap = conManager.conInfoMap(app.getConfiguration(),
@@ -276,7 +276,6 @@ public boolean reconfigure(int multiplier) {
 		Utils.printMemoryStatus();
 		return isRunning;
 	}
-
 	public void setDrainer(AbstractDrainer drainer) {
 		assert dp == null : "SNDrainProcessor has already been set";
 		this.dp = new SNDrainProcessorImpl(drainer);

From eda79993fb24e60290b28722b46435fdd7375392 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 07:30:43 +0800
Subject: [PATCH 687/881] Renamed:cfgManager->partitionManager.

---
 .../streamjit/impl/common/ConfigurationEditor.java |  6 +++---
 .../impl/distributed/DistributedBlobFactory.java   | 14 +++++++-------
 .../distributed/DistributedStreamCompiler.java     | 14 +++++++-------
 .../impl/distributed/StreamJitAppManager.java      |  6 +++---
 .../impl/distributed/runtimer/OnlineTuner.java     |  8 ++++----
 5 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index eddd6dae..aa2b71de 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -116,10 +116,10 @@ private static Pair<String, Integer> generate1(
 			OneToOneElement<?, ?> stream, int noOfnodes) {
 		StreamJitApp<?, ?> app = new StreamJitApp<>(stream);
 		int noofwrks = Workers.getIdentifier(app.sink) + 1;
-		PartitionManager cfgManager = new HotSpotTuning(app);
+		PartitionManager partitionManager = new HotSpotTuning(app);
 		ConnectionManager conManger = new BlockingTCPNoParams(0);
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManger,
-				noOfnodes);
+		BlobFactory bf = new DistributedBlobFactory(partitionManager,
+				conManger, noOfnodes);
 
 		Configuration cfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(app.source));
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 25ee1e17..d58d104b 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -35,20 +35,20 @@ public class DistributedBlobFactory implements BlobFactory {
 
 	private int noOfMachines;
 
-	private final PartitionManager cfgManager;
+	private final PartitionManager partitionManager;
 
 	private final ConnectionManager connectionManager;
 
-	public DistributedBlobFactory(PartitionManager cfgManager,
+	public DistributedBlobFactory(PartitionManager partitionManager,
 			ConnectionManager connectionManager, int noOfMachines) {
-		this.cfgManager = cfgManager;
+		this.partitionManager = partitionManager;
 		this.noOfMachines = noOfMachines;
 		this.connectionManager = connectionManager;
 	}
 
 	/**
-	 * If {@link PartitionManager} is not passed as a constructor argument
-	 * then {@link WorkerMachine} will be used as default one.
+	 * If {@link PartitionManager} is not passed as a constructor argument then
+	 * {@link WorkerMachine} will be used as default one.
 	 * 
 	 * @param noOfMachines
 	 */
@@ -66,8 +66,8 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
-		Configuration distCfg = cfgManager.getDefaultConfiguration(workers,
-				noOfMachines);
+		Configuration distCfg = partitionManager.getDefaultConfiguration(
+				workers, noOfMachines);
 		if (!GlobalConstants.useCompilerBlob)
 			return distCfg;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 7274f735..64b53c47 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -117,14 +117,14 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		StreamJitApp<I, O> app = new StreamJitApp<>(stream);
 		Controller controller = establishController();
 
-		PartitionManager cfgManager = new HotSpotTuning(app);
+		PartitionManager partitionManager = new HotSpotTuning(app);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
-		setConfiguration(controller, app, cfgManager, conManager);
+		setConfiguration(controller, app, partitionManager, conManager);
 
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
-				cfgManager, conManager, logger);
+				partitionManager, conManager, logger);
 		final AbstractDrainer drainer = new DistributedDrainer(app, logger,
 				manager);
 		drainer.setBlobGraph(app.blobGraph);
@@ -136,7 +136,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
-					cfgManager, logger, needTermination);
+					partitionManager, logger, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
 		}
 		return cs;
@@ -252,9 +252,9 @@ public void drain() {
 	}
 
 	private <I, O> void setConfiguration(Controller controller,
-			StreamJitApp<I, O> app, PartitionManager cfgManager,
+			StreamJitApp<I, O> app, PartitionManager partitionManager,
 			ConnectionManager conManager) {
-		BlobFactory bf = new DistributedBlobFactory(cfgManager, conManager,
+		BlobFactory bf = new DistributedBlobFactory(partitionManager, conManager,
 				Math.max(noOfnodes - 1, 1));
 		Configuration defaultCfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(app.source));
@@ -272,7 +272,7 @@ private <I, O> void setConfiguration(Controller controller,
 		} else
 			this.cfg = defaultCfg;
 
-		cfgManager.newConfiguration(this.cfg);
+		partitionManager.newConfiguration(this.cfg);
 	}
 
 	private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index af3b6f01..268e63ef 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -54,7 +54,7 @@ public class StreamJitAppManager {
 
 	private AppStatusProcessorImpl apStsPro = null;
 
-	private final PartitionManager cfgManager;
+	private final PartitionManager partitionManager;
 
 	private Map<Token, ConnectionInfo> conInfoMap;
 
@@ -108,11 +108,11 @@ public class StreamJitAppManager {
 	private final SNTimeInfoProcessor timeInfoProcessor;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
-			PartitionManager cfgManager, ConnectionManager conManager,
+			PartitionManager partitionManager, ConnectionManager conManager,
 			TimeLogger logger) {
 		this.controller = controller;
 		this.app = app;
-		this.cfgManager = cfgManager;
+		this.partitionManager = partitionManager;
 		this.conManager = conManager;
 		this.logger = logger;
 		this.timeInfoProcessor = new SNTimeInfoProcessorImpl(logger);
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e38d815d..d3992a53 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -32,18 +32,18 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitAppManager manager;
 	private final OpenTuner tuner;
 	private final StreamJitApp<?, ?> app;
-	private final PartitionManager cfgManager;
+	private final PartitionManager partitionManager;
 	private final boolean needTermination;
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
-			StreamJitApp<?, ?> app, PartitionManager cfgManager,
+			StreamJitApp<?, ?> app, PartitionManager partitionManager,
 			TimeLogger logger, boolean needTermination) {
 		this.drainer = drainer;
 		this.manager = manager;
 		this.app = app;
-		this.cfgManager = cfgManager;
+		this.partitionManager = partitionManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
 		this.logger = logger;
@@ -217,7 +217,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		if (manager.getStatus() == AppStatus.STOPPED)
 			return new Pair<Boolean, Long>(false, 0l);
 
-		if (!cfgManager.newConfiguration(config))
+		if (!partitionManager.newConfiguration(config))
 			return new Pair<Boolean, Long>(true, -2l);
 
 		if (!prognosticator.prognosticate(config))

From 20f572e25b1c4ba1ab2ed47a4596d37e4bdc3637 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 08:39:29 +0800
Subject: [PATCH 688/881] PartitionManger.getMachineWorkerMap() added.

---
 .../impl/distributed/HotSpotTuning.java       |  2 +-
 .../impl/distributed/PartitionManager.java    | 12 ++++++++++
 .../impl/distributed/WorkerMachine.java       | 24 +++++--------------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index ad72bee4..6422524a 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -64,7 +64,7 @@ public boolean newConfiguration(Configuration config) {
 		return true;
 	}
 
-	private Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
 			Configuration config) {
 		Map<Integer, Set<Worker<?, ?>>> partition = new HashMap<>();
 
diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index 5ced26d8..b166df53 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -58,6 +58,18 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 */
 	public boolean newConfiguration(Configuration config);
 
+	/**
+	 * Reads the configuration and returns a map of nodeID to list of set of
+	 * workers (list of blob workers). Value of the returned map is list of
+	 * worker set where each worker set is an individual blob.
+	 * 
+	 * @param config
+	 * @return map of nodeID to list of set of workers which are assigned to the
+	 *         node.
+	 */
+	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+			Configuration config);
+
 	/**
 	 * Implements the functions those can be called by runtimer to send
 	 * configuration information to streamnodes.
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 3899f678..5ce000e1 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -39,8 +39,11 @@
  */
 public final class WorkerMachine extends AbstractPartitionManager {
 
+	private final Set<Worker<?, ?>> workerset;
+
 	WorkerMachine(StreamJitApp app) {
 		super(app);
+		this.workerset = Workers.getAllWorkersInGraph(app.source);
 	}
 
 	@Override
@@ -85,8 +88,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	@Override
 	public boolean newConfiguration(Configuration config) {
 
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(
-				config, app.source);
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
 		try {
 			app.verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
@@ -96,22 +98,8 @@ public boolean newConfiguration(Configuration config) {
 		return true;
 	}
 
-	/**
-	 * Reads the configuration and returns a map of nodeID to list of set of
-	 * workers (list of blob workers) which are assigned to the node. Value of
-	 * the returned map is list of worker set where each worker set is an
-	 * individual blob.
-	 * 
-	 * @param config
-	 * @param workerset
-	 * @return map of nodeID to list of set of workers which are assigned to the
-	 *         node.
-	 */
-	private Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
-			Configuration config, Worker<?, ?> source) {
-
-		ImmutableSet<Worker<?, ?>> workerset = Workers
-				.getAllWorkersInGraph(source);
+	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+			Configuration config) {
 
 		Map<Integer, Set<Worker<?, ?>>> partition = new HashMap<>();
 		for (Worker<?, ?> w : workerset) {

From 49d7855219e2ed47c4977afbaf9c6d3538c10031 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 08:43:36 +0800
Subject: [PATCH 689/881] Renamed:getMachineWorkerMap->partitionMap()

PartitionManager.partitionMap.
---
 src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java    | 4 ++--
 src/edu/mit/streamjit/impl/distributed/PartitionManager.java | 2 +-
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java    | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 6422524a..7adf280f 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -54,7 +54,7 @@ public boolean newConfiguration(Configuration config) {
 		// System.out.println(p.getName() + " - Unknown type");
 		// }
 
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = partitionMap(config);
 		try {
 			app.verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
@@ -64,7 +64,7 @@ public boolean newConfiguration(Configuration config) {
 		return true;
 	}
 
-	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+	public Map<Integer, List<Set<Worker<?, ?>>>> partitionMap(
 			Configuration config) {
 		Map<Integer, Set<Worker<?, ?>>> partition = new HashMap<>();
 
diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index b166df53..deb35b4f 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -67,7 +67,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	 * @return map of nodeID to list of set of workers which are assigned to the
 	 *         node.
 	 */
-	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+	public Map<Integer, List<Set<Worker<?, ?>>>> partitionMap(
 			Configuration config);
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 5ce000e1..6b15e4d1 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -88,7 +88,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 	@Override
 	public boolean newConfiguration(Configuration config) {
 
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = getMachineWorkerMap(config);
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = partitionMap(config);
 		try {
 			app.verifyConfiguration(partitionsMachineMap);
 		} catch (StreamCompilationFailedException ex) {
@@ -98,7 +98,7 @@ public boolean newConfiguration(Configuration config) {
 		return true;
 	}
 
-	public Map<Integer, List<Set<Worker<?, ?>>>> getMachineWorkerMap(
+	public Map<Integer, List<Set<Worker<?, ?>>>> partitionMap(
 			Configuration config) {
 
 		Map<Integer, Set<Worker<?, ?>>> partition = new HashMap<>();

From 7da4c5fb52c08ecd958bd0bfb2907062ea4ed055 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 08:54:57 +0800
Subject: [PATCH 690/881] newConfiguration() moved to ConfigurationManager.

The method newConfiguration() has been moved from PartitionManager to
ConfigurationManager.
---
 .../distributed/ConfigurationManager.java     | 64 +++++++++++++++++++
 .../impl/distributed/HotSpotTuning.java       | 25 --------
 .../impl/distributed/PartitionManager.java    | 11 ----
 .../impl/distributed/WorkerMachine.java       | 30 ---------
 4 files changed, 64 insertions(+), 66 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java

diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
new file mode 100644
index 00000000..ef703fcc
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -0,0 +1,64 @@
+package edu.mit.streamjit.impl.distributed;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.mit.streamjit.api.StreamCompilationFailedException;
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+
+public class ConfigurationManager {
+
+	private final StreamJitApp<?, ?> app;
+
+	private final PartitionManager partitionManager;
+
+	public ConfigurationManager(StreamJitApp<?, ?> app,
+			PartitionManager partitionManager) {
+		this.app = app;
+		this.partitionManager = partitionManager;
+	}
+
+	/**
+	 * This method may be called to by the {@link OnlineTuner} to interpret a
+	 * new configuration and execute the steramjit app with the new
+	 * configuration.
+	 * <p>
+	 * Builds partitionsMachineMap and {@link BlobGraph} from the new
+	 * Configuration, and verifies for any cycles among blobs. If it is a valid
+	 * configuration, (i.e., no cycles among the blobs), then {@link #app}
+	 * object's member variables {@link StreamJitApp#blobConfiguration},
+	 * {@link StreamJitApp#blobGraph} and
+	 * {@link StreamJitApp#partitionsMachineMap} will be assigned according to
+	 * reflect the new configuration, no changes otherwise.
+	 * 
+	 * @param config
+	 *            configuration from {@link OnlineTuner}.
+	 * @return true iff valid configuration is passed.
+	 */
+	public boolean newConfiguration(Configuration config) {
+		// for (Parameter p : config.getParametersMap().values()) {
+		// if (p instanceof IntParameter) {
+		// IntParameter ip = (IntParameter) p;
+		// System.out.println(ip.getName() + " - " + ip.getValue());
+		// } else if (p instanceof SwitchParameter<?>) {
+		// SwitchParameter<?> sp = (SwitchParameter<?>) p;
+		// System.out.println(sp.getName() + " - " + sp.getValue());
+		// } else
+		// System.out.println(p.getName() + " - Unknown type");
+		// }
+
+		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = partitionManager
+				.partitionMap(config);
+		try {
+			app.verifyConfiguration(partitionsMachineMap);
+		} catch (StreamCompilationFailedException ex) {
+			return false;
+		}
+		app.setConfiguration(config);
+		return true;
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 7adf280f..022a0efa 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -13,7 +13,6 @@
 import edu.mit.streamjit.api.Pipeline;
 import edu.mit.streamjit.api.Splitjoin;
 import edu.mit.streamjit.api.Splitter;
-import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.StreamVisitor;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
@@ -40,30 +39,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 		return visitor.builder.build();
 	}
 
-	@Override
-	public boolean newConfiguration(Configuration config) {
-
-		// for (Parameter p : config.getParametersMap().values()) {
-		// if (p instanceof IntParameter) {
-		// IntParameter ip = (IntParameter) p;
-		// System.out.println(ip.getName() + " - " + ip.getValue());
-		// } else if (p instanceof SwitchParameter<?>) {
-		// SwitchParameter<?> sp = (SwitchParameter<?>) p;
-		// System.out.println(sp.getName() + " - " + sp.getValue());
-		// } else
-		// System.out.println(p.getName() + " - Unknown type");
-		// }
-
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = partitionMap(config);
-		try {
-			app.verifyConfiguration(partitionsMachineMap);
-		} catch (StreamCompilationFailedException ex) {
-			return false;
-		}
-		app.setConfiguration(config);
-		return true;
-	}
-
 	public Map<Integer, List<Set<Worker<?, ?>>>> partitionMap(
 			Configuration config) {
 		Map<Integer, Set<Worker<?, ?>>> partition = new HashMap<>();
diff --git a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
index deb35b4f..235c1b91 100644
--- a/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/PartitionManager.java
@@ -47,17 +47,6 @@ public interface PartitionManager {
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 			int noOfMachines);
 
-	/**
-	 * When opentuner gives a new configuration, this method may be called to
-	 * interpret the configuration and execute the steramjit app with the new
-	 * configuration.
-	 * 
-	 * @param config
-	 *            configuration from opentuner.
-	 * @return true iff valid configuration is passed.
-	 */
-	public boolean newConfiguration(Configuration config);
-
 	/**
 	 * Reads the configuration and returns a map of nodeID to list of set of
 	 * workers (list of blob workers). Value of the returned map is list of
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 6b15e4d1..6018a908 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -7,11 +7,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import com.google.common.collect.ImmutableSet;
-
-import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
@@ -72,32 +68,6 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 		return builder.build();
 	}
 
-	/**
-	 * Builds partitionsMachineMap and {@link BlobGraph} from the new
-	 * Configuration, and verifies for any cycles among blobs. If it is a valid
-	 * configuration, (i.e., no cycles among the blobs), then {@link #app}
-	 * object's member variables {@link StreamJitApp#blobConfiguration},
-	 * {@link StreamJitApp#blobGraph} and
-	 * {@link StreamJitApp#partitionsMachineMap} will be assigned according to
-	 * reflect the new configuration, no changes otherwise.
-	 * 
-	 * @param config
-	 *            New configuration form Opentuer.
-	 * @return true iff no cycles among blobs
-	 */
-	@Override
-	public boolean newConfiguration(Configuration config) {
-
-		Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap = partitionMap(config);
-		try {
-			app.verifyConfiguration(partitionsMachineMap);
-		} catch (StreamCompilationFailedException ex) {
-			return false;
-		}
-		app.setConfiguration(config);
-		return true;
-	}
-
 	public Map<Integer, List<Set<Worker<?, ?>>>> partitionMap(
 			Configuration config) {
 

From e3c371ba3211f5c8c00b295e79b62175f8f71f21 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 09:06:48 +0800
Subject: [PATCH 691/881] OnlineTuner uses CfgManager.newConfiguration()

---
 .../distributed/DistributedStreamCompiler.java   | 16 +++++++++-------
 .../impl/distributed/runtimer/OnlineTuner.java   | 10 +++++-----
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 64b53c47..9c738092 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -1,6 +1,5 @@
 package edu.mit.streamjit.impl.distributed;
 
-import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -118,9 +117,12 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		Controller controller = establishController();
 
 		PartitionManager partitionManager = new HotSpotTuning(app);
+		ConfigurationManager cfgManager = new ConfigurationManager(app,
+				partitionManager);
 		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
 				controller.controllerNodeID);
-		setConfiguration(controller, app, partitionManager, conManager);
+		setConfiguration(controller, app, partitionManager, conManager,
+				cfgManager);
 
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
@@ -136,7 +138,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		if (GlobalConstants.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
-					partitionManager, logger, needTermination);
+					cfgManager, logger, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
 		}
 		return cs;
@@ -253,9 +255,9 @@ public void drain() {
 
 	private <I, O> void setConfiguration(Controller controller,
 			StreamJitApp<I, O> app, PartitionManager partitionManager,
-			ConnectionManager conManager) {
-		BlobFactory bf = new DistributedBlobFactory(partitionManager, conManager,
-				Math.max(noOfnodes - 1, 1));
+			ConnectionManager conManager, ConfigurationManager cfgManager) {
+		BlobFactory bf = new DistributedBlobFactory(partitionManager,
+				conManager, Math.max(noOfnodes - 1, 1));
 		Configuration defaultCfg = bf.getDefaultConfiguration(Workers
 				.getAllWorkersInGraph(app.source));
 
@@ -272,7 +274,7 @@ private <I, O> void setConfiguration(Controller controller,
 		} else
 			this.cfg = defaultCfg;
 
-		partitionManager.newConfiguration(this.cfg);
+		cfgManager.newConfiguration(this.cfg);
 	}
 
 	private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d3992a53..f0aef929 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.TimeLogger;
-import edu.mit.streamjit.impl.distributed.PartitionManager;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
@@ -32,18 +32,18 @@ public class OnlineTuner implements Runnable {
 	private final StreamJitAppManager manager;
 	private final OpenTuner tuner;
 	private final StreamJitApp<?, ?> app;
-	private final PartitionManager partitionManager;
+	private final ConfigurationManager cfgManager;
 	private final boolean needTermination;
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
-			StreamJitApp<?, ?> app, PartitionManager partitionManager,
+			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
 			TimeLogger logger, boolean needTermination) {
 		this.drainer = drainer;
 		this.manager = manager;
 		this.app = app;
-		this.partitionManager = partitionManager;
+		this.cfgManager = cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
 		this.logger = logger;
@@ -217,7 +217,7 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		if (manager.getStatus() == AppStatus.STOPPED)
 			return new Pair<Boolean, Long>(false, 0l);
 
-		if (!partitionManager.newConfiguration(config))
+		if (!cfgManager.newConfiguration(config))
 			return new Pair<Boolean, Long>(true, -2l);
 
 		if (!prognosticator.prognosticate(config))

From b33132ef302c7023eb9ae943e1b4b81ac84b2dee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 11:38:41 +0800
Subject: [PATCH 692/881] AbstractDrainer has been moved to drainer package.

---
 .../streamjit/impl/common/{ => drainer}/AbstractDrainer.java  | 4 +++-
 src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java  | 2 +-
 .../streamjit/impl/concurrent/ConcurrentStreamCompiler.java   | 4 ++--
 .../mit/streamjit/impl/distributed/ConfigurationManager.java  | 2 +-
 .../streamjit/impl/distributed/DistributedStreamCompiler.java | 2 +-
 src/edu/mit/streamjit/impl/distributed/HeadChannel.java       | 2 +-
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java      | 2 +-
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java   | 2 +-
 .../streamjit/impl/distributed/common/GlobalConstants.java    | 2 +-
 .../impl/distributed/runtimer/DistributedDrainer.java         | 2 +-
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java  | 2 +-
 11 files changed, 14 insertions(+), 12 deletions(-)
 rename src/edu/mit/streamjit/impl/common/{ => drainer}/AbstractDrainer.java (99%)

diff --git a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
similarity index 99%
rename from src/edu/mit/streamjit/impl/common/AbstractDrainer.java
rename to src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index 668216b7..95f7ed86 100644
--- a/src/edu/mit/streamjit/impl/common/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -1,4 +1,4 @@
-package edu.mit.streamjit.impl.common;
+package edu.mit.streamjit.impl.common.drainer;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
@@ -36,6 +36,8 @@
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.common.IOInfo;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
index 614a3dea..62ce50c3 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
@@ -9,8 +9,8 @@
 
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.BlobThread;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
index 67e1990c..a1417335 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
@@ -23,8 +23,6 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.ConcurrentArrayBuffer;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
-import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.BlobThread;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -35,6 +33,8 @@
 import edu.mit.streamjit.impl.common.OutputBufferFactory;
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index ef703fcc..13bf5ccc 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -6,8 +6,8 @@
 
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
 public class ConfigurationManager {
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 9c738092..7284b7eb 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -22,12 +22,12 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.InputBufferFactory;
 import edu.mit.streamjit.impl.common.OutputBufferFactory;
 import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
diff --git a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
index be2a404a..b1dec0eb 100644
--- a/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/HeadChannel.java
@@ -4,7 +4,7 @@
 
 import edu.mit.streamjit.impl.blob.AbstractReadOnlyBuffer;
 import edu.mit.streamjit.impl.blob.Buffer;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.node.AsyncOutputChannel;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 85084cf6..ec7aaef0 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -25,7 +25,6 @@
 import edu.mit.streamjit.impl.blob.BlobFactory;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.common.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
@@ -34,6 +33,7 @@
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 268e63ef..24b55137 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -15,9 +15,9 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.AppStatus.AppStatusProcessor;
 import edu.mit.streamjit.impl.distributed.common.AsyncTCPConnection.AsyncTCPConnectionInfo;
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index aac57f1f..604884ed 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -1,6 +1,6 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import edu.mit.streamjit.impl.common.AbstractDrainer;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
index 701e4e55..c30da7ea 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/DistributedDrainer.java
@@ -1,8 +1,8 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
 import edu.mit.streamjit.impl.blob.Blob.Token;
-import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f0aef929..47a0c794 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -4,10 +4,10 @@
 import java.io.FileWriter;
 import java.io.IOException;
 
-import edu.mit.streamjit.impl.common.AbstractDrainer;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;

From a324e5fe45a3488f1a3f9960d6c19a3f4fc23b6f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 13:57:40 +0800
Subject: [PATCH 693/881] AbstractDrainer refers generic StreamJitApp.

---
 .../mit/streamjit/impl/common/drainer/AbstractDrainer.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index 95f7ed86..9b7e5225 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -120,9 +120,9 @@ public abstract class AbstractDrainer {
 
 	private final TimeLogger logger;
 
-	private final StreamJitApp app;
+	private final StreamJitApp<?, ?> app;
 
-	public AbstractDrainer(StreamJitApp app, TimeLogger logger) {
+	public AbstractDrainer(StreamJitApp<?, ?> app, TimeLogger logger) {
 		state = DrainerState.NODRAINING;
 		finalLatch = new CountDownLatch(1);
 		this.app = app;
@@ -390,6 +390,7 @@ private void printDrainDataStats(DrainData drainData) {
 			e.printStackTrace();
 		}
 	}
+
 	private void dumpDrainData(DrainData drainData) {
 		try {
 			String fileName = String.format("%s%sDrainData", app.name,

From ca725b9db716ff9b90ce7560059f602efe0c3283 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 14:23:30 +0800
Subject: [PATCH 694/881] AbstractDrainer has been refactored.

BlobGraph and BlobNode have been moved to a new file.
---
 .../impl/common/ConfigurationEditor.java      |   2 +-
 .../impl/common/drainer/AbstractDrainer.java  | 388 +----------------
 .../impl/common/drainer/BlobGraph.java        | 411 ++++++++++++++++++
 .../impl/concurrent/ConcurrentDrainer.java    |   1 +
 .../concurrent/ConcurrentStreamCompiler.java  |   2 +-
 .../distributed/ConfigurationManager.java     |   2 +-
 .../impl/distributed/StreamJitApp.java        |   2 +-
 7 files changed, 417 insertions(+), 391 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index aa2b71de..eb3527a3 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -16,11 +16,11 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
-import edu.mit.streamjit.impl.distributed.PartitionManager;
 import edu.mit.streamjit.impl.distributed.ConnectionManager;
 import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.DistributedBlobFactory;
 import edu.mit.streamjit.impl.distributed.HotSpotTuning;
+import edu.mit.streamjit.impl.distributed.PartitionManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.WorkerMachine;
 import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
diff --git a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index 9b7e5225..c15a9678 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -1,20 +1,14 @@
 package edu.mit.streamjit.impl.common.drainer;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -24,20 +18,16 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Sets;
 
 import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.Input;
-import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.StreamCompiler;
-import edu.mit.streamjit.api.Worker;
-import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.blob.DrainData;
-import edu.mit.streamjit.impl.common.IOInfo;
 import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.common.drainer.BlobGraph.BlobNode;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
@@ -565,382 +555,6 @@ private void drainingDone(BlobNode blobNode) {
 		}
 	}
 
-	/**
-	 * BlobGraph builds predecessor successor relationship for set of
-	 * partitioned workers, and verifies for cyclic dependencies among the
-	 * partitions. Blob graph doesn't keep blobs. Instead it keeps
-	 * {@link BlobNode} that represents blobs. </p> All BlobNodes in the graph
-	 * can be retrieved and used in coupled with {@link AbstractDrainer} to
-	 * successfully perform draining process.
-	 * 
-	 * @author Sumanan sumanan@mit.edu
-	 * @since Jul 30, 2013
-	 */
-	public static class BlobGraph {
-
-		/**
-		 * All nodes in the graph.
-		 */
-		private final ImmutableMap<Token, BlobNode> blobNodes;
-
-		/**
-		 * The blob which has the overall stream input.
-		 */
-		private final BlobNode sourceBlobNode;
-
-		public BlobGraph(List<Set<Worker<?, ?>>> partitionWorkers) {
-			checkNotNull(partitionWorkers);
-			Set<DummyBlob> blobSet = new HashSet<>();
-			for (Set<Worker<?, ?>> workers : partitionWorkers) {
-				blobSet.add(new DummyBlob(workers));
-			}
-
-			ImmutableMap.Builder<Token, BlobNode> builder = new ImmutableMap.Builder<>();
-			for (DummyBlob b : blobSet) {
-				builder.put(b.id, new BlobNode(b.id, b.inputs, b.outputs));
-			}
-
-			this.blobNodes = builder.build();
-
-			for (DummyBlob cur : blobSet) {
-				for (DummyBlob other : blobSet) {
-					if (cur == other)
-						continue;
-					if (Sets.intersection(cur.outputs, other.inputs).size() != 0) {
-						BlobNode curNode = blobNodes.get(cur.id);
-						BlobNode otherNode = blobNodes.get(other.id);
-
-						curNode.addSuccessor(otherNode);
-						otherNode.addPredecessor(curNode);
-					}
-				}
-			}
-
-			checkCycles(blobNodes.values());
-
-			BlobNode sourceBlob = null;
-			for (BlobNode bn : blobNodes.values()) {
-				if (bn.getDependencyCount() == 0) {
-					assert sourceBlob == null : "Multiple independent blobs found.";
-					sourceBlob = bn;
-				}
-			}
-
-			checkNotNull(sourceBlob);
-			this.sourceBlobNode = sourceBlob;
-		}
-
-		/**
-		 * @return BlobIds of all blobnodes in the blobgraph.
-		 */
-		public ImmutableSet<Token> getBlobIds() {
-			return blobNodes.keySet();
-		}
-
-		public BlobNode getBlobNode(Token blobID) {
-			return blobNodes.get(blobID);
-		}
-
-		/**
-		 * TODO: We may need to make the class {@link BlobNode} public and move
-		 * these functions to {@link BlobNode}.
-		 * <p>
-		 * Returns output edges of a blob. This method is added on [2014-03-01].
-		 * 
-		 * @param blobID
-		 * @return
-		 */
-		public ImmutableSet<Token> getOutputs(Token blobID) {
-			return blobNodes.get(blobID).outputs;
-		}
-
-		/**
-		 * TODO: We may need to make the class {@link BlobNode} public and move
-		 * these functions to {@link BlobNode}.
-		 * <p>
-		 * Returns input edges of a blob. This method is added on [2014-03-01].
-		 * 
-		 * @param blobID
-		 * @return
-		 */
-		public ImmutableSet<Token> getInputs(Token blobID) {
-			return blobNodes.get(blobID).inputs;
-		}
-
-		/**
-		 * A Drainer can be set to the {@link BlobGraph} to perform draining.
-		 * 
-		 * @param drainer
-		 */
-		public void setDrainer(AbstractDrainer drainer) {
-			for (BlobNode bn : blobNodes.values()) {
-				bn.setDrainer(drainer);
-			}
-		}
-
-		public void clearDrainData() {
-			for (BlobNode node : blobNodes.values()) {
-				node.snDrainData = null;
-			}
-		}
-
-		/**
-		 * @return the sourceBlobNode
-		 */
-		private BlobNode getSourceBlobNode() {
-			return sourceBlobNode;
-		}
-
-		/**
-		 * Does a depth first traversal to detect cycles in the graph.
-		 * 
-		 * @param blobNodes
-		 */
-		private void checkCycles(Collection<BlobNode> blobNodes) {
-			Map<BlobNode, Color> colorMap = new HashMap<>();
-			for (BlobNode b : blobNodes) {
-				colorMap.put(b, Color.WHITE);
-			}
-			for (BlobNode b : blobNodes) {
-				if (colorMap.get(b) == Color.WHITE)
-					if (DFS(b, colorMap))
-						throw new StreamCompilationFailedException(
-								"Cycles found among blobs");
-			}
-		}
-
-		/**
-		 * A cycle exits in a directed graph if a back edge is detected during a
-		 * DFS traversal. A back edge exists in a directed graph if the
-		 * currently explored vertex has an adjacent vertex that was already
-		 * colored gray
-		 * 
-		 * @param vertex
-		 * @param colorMap
-		 * @return <code>true</code> if cycle found, <code>false</code>
-		 *         otherwise.
-		 */
-		private boolean DFS(BlobNode vertex, Map<BlobNode, Color> colorMap) {
-			colorMap.put(vertex, Color.GRAY);
-			for (BlobNode adj : vertex.getSuccessors()) {
-				if (colorMap.get(adj) == Color.GRAY)
-					return true;
-				if (colorMap.get(adj) == Color.WHITE)
-					if (DFS(adj, colorMap))
-						return true;
-			}
-			colorMap.put(vertex, Color.BLACK);
-			return false;
-		}
-
-		/**
-		 * Just used to build the input and output tokens of a partitioned blob
-		 * workers. imitate a {@link Blob}.
-		 */
-		private final class DummyBlob {
-			private final ImmutableSet<Token> inputs;
-			private final ImmutableSet<Token> outputs;
-			private final Token id;
-
-			private DummyBlob(Set<Worker<?, ?>> workers) {
-				ImmutableSet.Builder<Token> inputBuilder = new ImmutableSet.Builder<>();
-				ImmutableSet.Builder<Token> outputBuilder = new ImmutableSet.Builder<>();
-				for (IOInfo info : IOInfo.externalEdges(workers)) {
-					(info.isInput() ? inputBuilder : outputBuilder).add(info
-							.token());
-				}
-
-				inputs = inputBuilder.build();
-				outputs = outputBuilder.build();
-				id = Collections.min(inputs);
-			}
-		}
-	}
-
-	/**
-	 * BlobNode represents the vertex in the blob graph ({@link BlobGraph}). It
-	 * represents a {@link Blob} and carry the draining process of that blob.
-	 * 
-	 * @author Sumanan
-	 */
-	private static final class BlobNode {
-
-		/**
-		 * Intermediate drain data.
-		 */
-		private SNDrainedData snDrainData;
-
-		private AbstractDrainer drainer;
-		/**
-		 * The blob that wrapped by this blob node.
-		 */
-		private final Token blobID;
-		/**
-		 * Predecessor blob nodes of this blob node.
-		 */
-		private List<BlobNode> predecessors;
-		/**
-		 * Successor blob nodes of this blob node.
-		 */
-		private List<BlobNode> successors;
-		/**
-		 * The number of undrained predecessors of this blobs. Everytime, when a
-		 * predecessor finished draining, dependencyCount will be decremented
-		 * and once it reached to 0 this blob will be called for draining.
-		 */
-		private AtomicInteger dependencyCount;
-
-		// TODO: add comments
-		private AtomicInteger drainState;
-
-		/**
-		 * All input channels of this blob. We need this information to globally
-		 * determine buffer sizes to avoid deadlocks. This is added on
-		 * [2014-03-01], when implementing global buffer size adjustment.
-		 */
-		private final ImmutableSet<Token> inputs;
-
-		/**
-		 * All output channels of this blob. We need this information to
-		 * globally determine buffer sizes to avoid deadlocks. This is added on
-		 * [2014-03-01], when implementing global buffer size adjustment.
-		 */
-		private final ImmutableSet<Token> outputs;
-
-		private BlobNode(Token blob, ImmutableSet<Token> inputs,
-				ImmutableSet<Token> outputs) {
-			this.blobID = blob;
-			predecessors = new ArrayList<>();
-			successors = new ArrayList<>();
-			dependencyCount = new AtomicInteger(0);
-			drainState = new AtomicInteger(0);
-			this.inputs = inputs;
-			this.outputs = outputs;
-		}
-
-		/**
-		 * Should be called when the draining of the current blob has been
-		 * finished. This function stops all threads belong to the blob and
-		 * inform its successors as well.
-		 */
-		private void drained() {
-			if (drainState.compareAndSet(1, 3)) {
-				for (BlobNode suc : this.successors) {
-					suc.predecessorDrained(this);
-				}
-				drainer.drainingDone(this);
-			} else if (drainState.compareAndSet(2, 3)) {
-				drainer.drainingDone(this);
-			}
-		}
-
-		/**
-		 * Drain the blob mapped by this blob node.
-		 */
-		private void drain() {
-			checkNotNull(drainer);
-			if (!drainState.compareAndSet(0, 1)) {
-				throw new IllegalStateException(
-						"Drain of this blobNode has already been called");
-			}
-
-			DrainType drainType;
-			if (GlobalConstants.useDrainData)
-				if (drainer.state == DrainerState.FINAL)
-					drainType = DrainType.FINAL;
-				else
-					drainType = DrainType.INTERMEDIATE;
-			else
-				drainType = DrainType.DISCARD;
-
-			drainer.drain(blobID, drainType);
-
-			// TODO: Verify the waiting time is reasonable.
-			if (GlobalConstants.needDrainDeadlockHandler)
-				drainer.schExecutorService.schedule(deadLockHandler(), 6000,
-						TimeUnit.MILLISECONDS);
-		}
-
-		private void setDrainData(SNDrainedData drainedData) {
-			if (this.snDrainData == null) {
-				this.snDrainData = drainedData;
-				drainState.set(4);
-			} else
-				throw new AssertionError(
-						"Multiple drain data has been received.");
-		}
-
-		private ImmutableList<BlobNode> getSuccessors() {
-			return ImmutableList.copyOf(successors);
-		}
-
-		private void addPredecessor(BlobNode pred) {
-			assert !predecessors.contains(pred) : String.format(
-					"The BlobNode %s has already been set as a predecessors",
-					pred);
-			predecessors.add(pred);
-			dependencyCount.set(dependencyCount.get() + 1);
-		}
-
-		private void addSuccessor(BlobNode succ) {
-			assert !successors.contains(succ) : String
-					.format("The BlobNode %s has already been set as a successor",
-							succ);
-			successors.add(succ);
-		}
-
-		private void predecessorDrained(BlobNode pred) {
-			if (!predecessors.contains(pred))
-				throw new IllegalArgumentException("Illegal Predecessor");
-
-			assert dependencyCount.get() > 0 : String
-					.format("Graph mismatch : My predecessors count is %d. But more than %d of BlobNodes claim me as their successor",
-							predecessors.size(), predecessors.size());
-
-			if (dependencyCount.decrementAndGet() == 0) {
-				drain();
-			}
-		}
-
-		/**
-		 * @return The number of undrained predecessors.
-		 */
-		private int getDependencyCount() {
-			return dependencyCount.get();
-		}
-
-		private void setDrainer(AbstractDrainer drainer) {
-			checkNotNull(drainer);
-			this.drainer = drainer;
-		}
-
-		private Runnable deadLockHandler() {
-			Runnable r = new Runnable() {
-
-				@Override
-				public void run() {
-					if (drainState.compareAndSet(1, 2)) {
-						for (BlobNode suc : successors) {
-							suc.predecessorDrained(BlobNode.this);
-						}
-						System.out
-								.println("deadLockHandler: "
-										+ blobID
-										+ " - Deadlock during draining has been handled");
-					}
-				}
-			};
-			return r;
-		}
-	}
-
-	/**
-	 * Color enumerator used by DFS algorithm to find cycles in the blob graph.
-	 */
-	private enum Color {
-		WHITE, GRAY, BLACK
-	}
-
 	/**
 	 * Reflects {@link AbstractDrainer}'s state.
 	 */
diff --git a/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
new file mode 100644
index 00000000..f8663bb2
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
@@ -0,0 +1,411 @@
+package edu.mit.streamjit.impl.common.drainer;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import edu.mit.streamjit.api.StreamCompilationFailedException;
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob;
+import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.common.IOInfo;
+import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
+
+/**
+ * [14 Feb, 2015] This class was an inner class of {@link AbstractDrainer}. I
+ * have re factored {@link AbstractDrainer} and moved this class a new file.
+ * 
+ * BlobGraph builds predecessor successor relationship for set of partitioned
+ * workers, and verifies for cyclic dependencies among the partitions. Blob
+ * graph doesn't keep blobs. Instead it keeps {@link BlobNode} that represents
+ * blobs. </p> All BlobNodes in the graph can be retrieved and used in coupled
+ * with {@link AbstractDrainer} to successfully perform draining process.
+ * 
+ * @author Sumanan sumanan@mit.edu
+ * @since Jul 30, 2013
+ */
+public class BlobGraph {
+
+	/**
+	 * All nodes in the graph.
+	 */
+	final ImmutableMap<Token, BlobNode> blobNodes;
+
+	/**
+	 * The blob which has the overall stream input.
+	 */
+	private final BlobNode sourceBlobNode;
+
+	public BlobGraph(List<Set<Worker<?, ?>>> partitionWorkers) {
+		checkNotNull(partitionWorkers);
+		Set<DummyBlob> blobSet = new HashSet<>();
+		for (Set<Worker<?, ?>> workers : partitionWorkers) {
+			blobSet.add(new DummyBlob(workers));
+		}
+
+		ImmutableMap.Builder<Token, BlobNode> builder = new ImmutableMap.Builder<>();
+		for (DummyBlob b : blobSet) {
+			builder.put(b.id, new BlobNode(b.id, b.inputs, b.outputs));
+		}
+
+		this.blobNodes = builder.build();
+
+		for (DummyBlob cur : blobSet) {
+			for (DummyBlob other : blobSet) {
+				if (cur == other)
+					continue;
+				if (Sets.intersection(cur.outputs, other.inputs).size() != 0) {
+					BlobNode curNode = blobNodes.get(cur.id);
+					BlobNode otherNode = blobNodes.get(other.id);
+
+					curNode.addSuccessor(otherNode);
+					otherNode.addPredecessor(curNode);
+				}
+			}
+		}
+
+		checkCycles(blobNodes.values());
+
+		BlobNode sourceBlob = null;
+		for (BlobNode bn : blobNodes.values()) {
+			if (bn.getDependencyCount() == 0) {
+				assert sourceBlob == null : "Multiple independent blobs found.";
+				sourceBlob = bn;
+			}
+		}
+
+		checkNotNull(sourceBlob);
+		this.sourceBlobNode = sourceBlob;
+	}
+
+	/**
+	 * @return BlobIds of all blobnodes in the blobgraph.
+	 */
+	public ImmutableSet<Token> getBlobIds() {
+		return blobNodes.keySet();
+	}
+
+	public BlobNode getBlobNode(Token blobID) {
+		return blobNodes.get(blobID);
+	}
+
+	/**
+	 * TODO: We may need to make the class {@link BlobNode} public and move
+	 * these functions to {@link BlobNode}.
+	 * <p>
+	 * Returns output edges of a blob. This method is added on [2014-03-01].
+	 * 
+	 * @param blobID
+	 * @return
+	 */
+	public ImmutableSet<Token> getOutputs(Token blobID) {
+		return blobNodes.get(blobID).outputs;
+	}
+
+	/**
+	 * TODO: We may need to make the class {@link BlobNode} public and move
+	 * these functions to {@link BlobNode}.
+	 * <p>
+	 * Returns input edges of a blob. This method is added on [2014-03-01].
+	 * 
+	 * @param blobID
+	 * @return
+	 */
+	public ImmutableSet<Token> getInputs(Token blobID) {
+		return blobNodes.get(blobID).inputs;
+	}
+
+	/**
+	 * A Drainer can be set to the {@link BlobGraph} to perform draining.
+	 * 
+	 * @param drainer
+	 */
+	public void setDrainer(AbstractDrainer drainer) {
+		for (BlobNode bn : blobNodes.values()) {
+			bn.setDrainer(drainer);
+		}
+	}
+
+	public void clearDrainData() {
+		for (BlobNode node : blobNodes.values()) {
+			node.snDrainData = null;
+		}
+	}
+
+	/**
+	 * @return the sourceBlobNode
+	 */
+	BlobNode getSourceBlobNode() {
+		return sourceBlobNode;
+	}
+
+	/**
+	 * Does a depth first traversal to detect cycles in the graph.
+	 * 
+	 * @param blobNodes
+	 */
+	private void checkCycles(Collection<BlobNode> blobNodes) {
+		Map<BlobNode, Color> colorMap = new HashMap<>();
+		for (BlobNode b : blobNodes) {
+			colorMap.put(b, Color.WHITE);
+		}
+		for (BlobNode b : blobNodes) {
+			if (colorMap.get(b) == Color.WHITE)
+				if (DFS(b, colorMap))
+					throw new StreamCompilationFailedException(
+							"Cycles found among blobs");
+		}
+	}
+
+	/**
+	 * A cycle exits in a directed graph if a back edge is detected during a DFS
+	 * traversal. A back edge exists in a directed graph if the currently
+	 * explored vertex has an adjacent vertex that was already colored gray
+	 * 
+	 * @param vertex
+	 * @param colorMap
+	 * @return <code>true</code> if cycle found, <code>false</code> otherwise.
+	 */
+	private boolean DFS(BlobNode vertex, Map<BlobNode, Color> colorMap) {
+		colorMap.put(vertex, Color.GRAY);
+		for (BlobNode adj : vertex.getSuccessors()) {
+			if (colorMap.get(adj) == Color.GRAY)
+				return true;
+			if (colorMap.get(adj) == Color.WHITE)
+				if (DFS(adj, colorMap))
+					return true;
+		}
+		colorMap.put(vertex, Color.BLACK);
+		return false;
+	}
+
+	/**
+	 * Just used to build the input and output tokens of a partitioned blob
+	 * workers. imitate a {@link Blob}.
+	 */
+	private final class DummyBlob {
+		private final ImmutableSet<Token> inputs;
+		private final ImmutableSet<Token> outputs;
+		private final Token id;
+
+		private DummyBlob(Set<Worker<?, ?>> workers) {
+			ImmutableSet.Builder<Token> inputBuilder = new ImmutableSet.Builder<>();
+			ImmutableSet.Builder<Token> outputBuilder = new ImmutableSet.Builder<>();
+			for (IOInfo info : IOInfo.externalEdges(workers)) {
+				(info.isInput() ? inputBuilder : outputBuilder).add(info
+						.token());
+			}
+
+			inputs = inputBuilder.build();
+			outputs = outputBuilder.build();
+			id = Collections.min(inputs);
+		}
+	}
+
+	/**
+	 * [14 Feb, 2015] This class was an inner class of {@link AbstractDrainer}.
+	 * I have re factored {@link AbstractDrainer} and moved this class to here.
+	 * {@link AbstractDrainer} directly accessed lots of fields and methods of
+	 * this class when this was an inner class of it. So those fields and
+	 * methods of this class have been made as package private when re
+	 * factoring. </p>
+	 * 
+	 * BlobNode represents the vertex in the blob graph ({@link BlobGraph} ). It
+	 * represents a {@link Blob} and carry the draining process of that blob.
+	 * 
+	 * @author Sumanan
+	 */
+	static final class BlobNode {
+
+		/**
+		 * Intermediate drain data.
+		 */
+		SNDrainedData snDrainData;
+
+		private AbstractDrainer drainer;
+		/**
+		 * The blob that wrapped by this blob node.
+		 */
+		final Token blobID;
+		/**
+		 * Predecessor blob nodes of this blob node.
+		 */
+		private List<BlobNode> predecessors;
+		/**
+		 * Successor blob nodes of this blob node.
+		 */
+		private List<BlobNode> successors;
+		/**
+		 * The number of undrained predecessors of this blobs. Everytime, when a
+		 * predecessor finished draining, dependencyCount will be decremented
+		 * and once it reached to 0 this blob will be called for draining.
+		 */
+		private AtomicInteger dependencyCount;
+
+		// TODO: add comments
+		AtomicInteger drainState;
+
+		/**
+		 * All input channels of this blob. We need this information to globally
+		 * determine buffer sizes to avoid deadlocks. This is added on
+		 * [2014-03-01], when implementing global buffer size adjustment.
+		 */
+		private final ImmutableSet<Token> inputs;
+
+		/**
+		 * All output channels of this blob. We need this information to
+		 * globally determine buffer sizes to avoid deadlocks. This is added on
+		 * [2014-03-01], when implementing global buffer size adjustment.
+		 */
+		private final ImmutableSet<Token> outputs;
+
+		private BlobNode(Token blob, ImmutableSet<Token> inputs,
+				ImmutableSet<Token> outputs) {
+			this.blobID = blob;
+			predecessors = new ArrayList<>();
+			successors = new ArrayList<>();
+			dependencyCount = new AtomicInteger(0);
+			drainState = new AtomicInteger(0);
+			this.inputs = inputs;
+			this.outputs = outputs;
+		}
+
+		/**
+		 * Should be called when the draining of the current blob has been
+		 * finished. This function stops all threads belong to the blob and
+		 * inform its successors as well.
+		 */
+		void drained() {
+			if (drainState.compareAndSet(1, 3)) {
+				for (BlobNode suc : this.successors) {
+					suc.predecessorDrained(this);
+				}
+				drainer.drainingDone(this);
+			} else if (drainState.compareAndSet(2, 3)) {
+				drainer.drainingDone(this);
+			}
+		}
+
+		/**
+		 * Drain the blob mapped by this blob node.
+		 */
+		void drain() {
+			checkNotNull(drainer);
+			if (!drainState.compareAndSet(0, 1)) {
+				throw new IllegalStateException(
+						"Drain of this blobNode has already been called");
+			}
+
+			DrainType drainType;
+			if (GlobalConstants.useDrainData)
+				if (drainer.state == DrainerState.FINAL)
+					drainType = DrainType.FINAL;
+				else
+					drainType = DrainType.INTERMEDIATE;
+			else
+				drainType = DrainType.DISCARD;
+
+			drainer.drain(blobID, drainType);
+
+			// TODO: Verify the waiting time is reasonable.
+			if (GlobalConstants.needDrainDeadlockHandler)
+				drainer.schExecutorService.schedule(deadLockHandler(), 6000,
+						TimeUnit.MILLISECONDS);
+		}
+
+		void setDrainData(SNDrainedData drainedData) {
+			if (this.snDrainData == null) {
+				this.snDrainData = drainedData;
+				drainState.set(4);
+			} else
+				throw new AssertionError(
+						"Multiple drain data has been received.");
+		}
+
+		private ImmutableList<BlobNode> getSuccessors() {
+			return ImmutableList.copyOf(successors);
+		}
+
+		private void addPredecessor(BlobNode pred) {
+			assert !predecessors.contains(pred) : String.format(
+					"The BlobNode %s has already been set as a predecessors",
+					pred);
+			predecessors.add(pred);
+			dependencyCount.set(dependencyCount.get() + 1);
+		}
+
+		private void addSuccessor(BlobNode succ) {
+			assert !successors.contains(succ) : String
+					.format("The BlobNode %s has already been set as a successor",
+							succ);
+			successors.add(succ);
+		}
+
+		private void predecessorDrained(BlobNode pred) {
+			if (!predecessors.contains(pred))
+				throw new IllegalArgumentException("Illegal Predecessor");
+
+			assert dependencyCount.get() > 0 : String
+					.format("Graph mismatch : My predecessors count is %d. But more than %d of BlobNodes claim me as their successor",
+							predecessors.size(), predecessors.size());
+
+			if (dependencyCount.decrementAndGet() == 0) {
+				drain();
+			}
+		}
+
+		/**
+		 * @return The number of undrained predecessors.
+		 */
+		private int getDependencyCount() {
+			return dependencyCount.get();
+		}
+
+		private void setDrainer(AbstractDrainer drainer) {
+			checkNotNull(drainer);
+			this.drainer = drainer;
+		}
+
+		private Runnable deadLockHandler() {
+			Runnable r = new Runnable() {
+
+				@Override
+				public void run() {
+					if (drainState.compareAndSet(1, 2)) {
+						for (BlobNode suc : successors) {
+							suc.predecessorDrained(BlobNode.this);
+						}
+						System.out
+								.println("deadLockHandler: "
+										+ blobID
+										+ " - Deadlock during draining has been handled");
+					}
+				}
+			};
+			return r;
+		}
+	}
+
+	/**
+	 * Color enumerator used by DFS algorithm to find cycles in the blob graph.
+	 */
+	private enum Color {
+		WHITE, GRAY, BLACK
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
index 62ce50c3..75e49d03 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
@@ -11,6 +11,7 @@
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.BlobThread;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
+import edu.mit.streamjit.impl.common.drainer.BlobGraph;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
index a1417335..031e595d 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
@@ -34,7 +34,7 @@
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
-import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
+import edu.mit.streamjit.impl.common.drainer.BlobGraph;
 import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index 13bf5ccc..de297756 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -7,7 +7,7 @@
 import edu.mit.streamjit.api.StreamCompilationFailedException;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
+import edu.mit.streamjit.impl.common.drainer.BlobGraph;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 
 public class ConfigurationManager {
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index ec7aaef0..96fcde99 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -33,7 +33,7 @@
 import edu.mit.streamjit.impl.common.Portals;
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.BlobGraph;
+import edu.mit.streamjit.impl.common.drainer.BlobGraph;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;

From 76b882813bf94f4ce9004c736451ae0408d43918 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 14:59:57 +0800
Subject: [PATCH 695/881] AbsDriner.state&schExecutorService package visible

AbstractDrainer.schExecutorService and AbstractDrainer.state have been made
package private during the re factoring. We can make those fields private by
moving BlobNode.drain() and BlobNode.drained() to AbstractDrainer.
---
 .../streamjit/impl/common/drainer/AbstractDrainer.java    | 8 ++++----
 src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java  | 8 ++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index c15a9678..dbe72bbc 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -101,12 +101,12 @@ public abstract class AbstractDrainer {
 
 	private AtomicInteger unDrainedNodes;
 
-	private ScheduledExecutorService schExecutorService;
+	ScheduledExecutorService schExecutorService;
 
 	/**
 	 * State of the drainer.
 	 */
-	private DrainerState state;
+	DrainerState state;
 
 	private final TimeLogger logger;
 
@@ -540,7 +540,7 @@ public final void awaitDrained(long timeout, TimeUnit unit)
 	 * 
 	 * @param blobNode
 	 */
-	private void drainingDone(BlobNode blobNode) {
+	void drainingDone(BlobNode blobNode) {
 		assert state != DrainerState.NODRAINING : "Illegal call. Drainer is not in draining mode.";
 		drainingDone(blobNode.blobID, state == DrainerState.FINAL);
 		if (unDrainedNodes.decrementAndGet() == 0) {
@@ -558,7 +558,7 @@ private void drainingDone(BlobNode blobNode) {
 	/**
 	 * Reflects {@link AbstractDrainer}'s state.
 	 */
-	private enum DrainerState {
+	enum DrainerState {
 		NODRAINING, /**
 		 * Draining in middle of the stream graph's execution. This
 		 * type of draining will be triggered by the open tuner for
diff --git a/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
index f8663bb2..73e3117c 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
@@ -23,6 +23,7 @@
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.IOInfo;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.DrainerState;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
@@ -225,6 +226,13 @@ private DummyBlob(Set<Worker<?, ?>> workers) {
 	 * this class when this was an inner class of it. So those fields and
 	 * methods of this class have been made as package private when re
 	 * factoring. </p>
+	 * <p>
+	 * [14 Feb, 2015] TODO: {@link AbstractDrainer#schExecutorService} and
+	 * {@link AbstractDrainer#state} have been made package private during the
+	 * re factoring. We can make those fields private by moving
+	 * {@link BlobNode#drain()} and {@link BlobNode#drained()} to
+	 * {@link AbstractDrainer}.
+	 * </p>
 	 * 
 	 * BlobNode represents the vertex in the blob graph ({@link BlobGraph} ). It
 	 * represents a {@link Blob} and carry the draining process of that blob.

From 8081fcd43212521ada3ef1dba049785fd59e425c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 20:02:28 +0800
Subject: [PATCH 696/881] field partitionManage removed from SJitAppManager

Unused field.
---
 .../impl/distributed/DistributedStreamCompiler.java         | 2 +-
 .../mit/streamjit/impl/distributed/StreamJitAppManager.java | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 7284b7eb..84214e1d 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -126,7 +126,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 
 		TimeLogger logger = new TimeLoggers.FileTimeLogger(app.name);
 		StreamJitAppManager manager = new StreamJitAppManager(controller, app,
-				partitionManager, conManager, logger);
+				conManager, logger);
 		final AbstractDrainer drainer = new DistributedDrainer(app, logger,
 				manager);
 		drainer.setBlobGraph(app.blobGraph);
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 24b55137..5e9a6bda 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -54,8 +54,6 @@ public class StreamJitAppManager {
 
 	private AppStatusProcessorImpl apStsPro = null;
 
-	private final PartitionManager partitionManager;
-
 	private Map<Token, ConnectionInfo> conInfoMap;
 
 	private final ConnectionManager conManager;
@@ -108,11 +106,9 @@ public class StreamJitAppManager {
 	private final SNTimeInfoProcessor timeInfoProcessor;
 
 	public StreamJitAppManager(Controller controller, StreamJitApp<?, ?> app,
-			PartitionManager partitionManager, ConnectionManager conManager,
-			TimeLogger logger) {
+			ConnectionManager conManager, TimeLogger logger) {
 		this.controller = controller;
 		this.app = app;
-		this.partitionManager = partitionManager;
 		this.conManager = conManager;
 		this.logger = logger;
 		this.timeInfoProcessor = new SNTimeInfoProcessorImpl(logger);

From 35141a57690429e58a51dc935fd43892f7d4a555 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 20:27:57 +0800
Subject: [PATCH 697/881] Renamed:coresPerMachine->machineCoreMap.

Bcz PartitionParameter uses "machineCoreMap" to refer this variable. So lets
use the same name.
---
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 96fcde99..c627db9a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -377,14 +377,14 @@ public Configuration getStaticConfiguration() {
 	public Configuration getDynamicConfiguration() {
 		Configuration.Builder builder = Configuration.builder();
 
-		Map<Integer, Integer> coresPerMachine = new HashMap<>();
+		Map<Integer, Integer> machineCoreMap = new HashMap<>();
 		for (Entry<Integer, List<Set<Worker<?, ?>>>> machine : partitionsMachineMap
 				.entrySet()) {
-			coresPerMachine.put(machine.getKey(), machine.getValue().size());
+			machineCoreMap.put(machine.getKey(), machine.getValue().size());
 		}
 
 		PartitionParameter.Builder partParam = PartitionParameter.builder(
-				GlobalConstants.PARTITION, coresPerMachine);
+				GlobalConstants.PARTITION, machineCoreMap);
 
 		BlobFactory intFactory = new Interpreter.InterpreterBlobFactory();
 		BlobFactory comp2Factory = new Compiler2BlobFactory();

From 1623560c32388415ab690d61f5159105d7d33805 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 20:50:22 +0800
Subject: [PATCH 698/881] BlobSpecifier carries maxCores for each blob.

In past, StreamNodes used GlobalConstants.maxNumCores to create blobs. Now
they get the maxNumCores from the configuration's blob specifier.
---
 .../mit/streamjit/impl/distributed/StreamJitApp.java  | 11 +++++++++--
 .../distributed/node/ConfigurationProcessorImpl.java  |  4 ++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index c627db9a..25beac2a 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -377,10 +377,13 @@ public Configuration getStaticConfiguration() {
 	public Configuration getDynamicConfiguration() {
 		Configuration.Builder builder = Configuration.builder();
 
+		int maxCores = maxCores();
+
 		Map<Integer, Integer> machineCoreMap = new HashMap<>();
 		for (Entry<Integer, List<Set<Worker<?, ?>>>> machine : partitionsMachineMap
 				.entrySet()) {
-			machineCoreMap.put(machine.getKey(), machine.getValue().size());
+			machineCoreMap.put(machine.getKey(), machine.getValue().size()
+					* maxCores);
 		}
 
 		PartitionParameter.Builder partParam = PartitionParameter.builder(
@@ -399,7 +402,7 @@ public Configuration getDynamicConfiguration() {
 					.get(machineID);
 			for (Set<Worker<?, ?>> blobWorkers : blobList) {
 				// TODO: One core per blob. Need to change this.
-				partParam.addBlob(machineID, 1, bf, blobWorkers);
+				partParam.addBlob(machineID, maxCores, bf, blobWorkers);
 
 				// TODO: Temp fix to build.
 				Token t = Utils.getblobID(blobWorkers);
@@ -426,4 +429,8 @@ private Configuration getInterpreterConfg() {
 		builder.addParameter(cfParameter);
 		return builder.build();
 	}
+
+	private int maxCores() {
+		return GlobalConstants.maxNumCores;
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index f712efcd..279588e8 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -145,11 +145,11 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
 				try {
 					BlobFactory bf = bs.getBlobFactory();
+					int maxCores = bs.getCores();
 					Stopwatch sw = Stopwatch.createStarted();
 					DrainData dd = drainData == null ? null : drainData
 							.subset(workIdentifiers);
-					Blob b = bf.makeBlob(workerset, blobConfigs,
-							GlobalConstants.maxNumCores, dd);
+					Blob b = bf.makeBlob(workerset, blobConfigs, maxCores, dd);
 					sendCompilationTime(sw, Utils.getblobID(workerset));
 					blobSet.add(b);
 				} catch (Exception ex) {

From 92723707f409b729bfd31eedfae7ff09b3c964cc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 14 Feb 2015 23:46:17 +0800
Subject: [PATCH 699/881] Tunes maximum number of cores.

---
 .../impl/distributed/DistributedBlobFactory.java       | 10 ++++++++++
 .../mit/streamjit/impl/distributed/StreamJitApp.java   |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index d58d104b..a74ea020 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -78,9 +78,19 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			builder.addParameter(p);
 
 		connectionManager.addConnectionParameters(builder, workers);
+		addMaxCoreParam(builder);
 		return builder.build();
 	}
 
+	private void addMaxCoreParam(Configuration.Builder builder) {
+		int min = 1;
+		int val = GlobalConstants.maxNumCores / 2;
+		val = min > val ? min : val;
+		Parameter p = new Configuration.IntParameter("maxNumCores", min,
+				GlobalConstants.maxNumCores, val);
+		builder.addParameter(p);
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		return getClass() == o.getClass()
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 25beac2a..2ceda404 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -26,6 +26,7 @@
 import edu.mit.streamjit.impl.blob.Buffer;
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.Configuration.PartitionParameter;
 import edu.mit.streamjit.impl.common.Configuration.SwitchParameter;
 import edu.mit.streamjit.impl.common.ConnectWorkersVisitor;
@@ -431,6 +432,10 @@ private Configuration getInterpreterConfg() {
 	}
 
 	private int maxCores() {
+		IntParameter maxCoreParam = configuration.getParameter("maxNumCores",
+				IntParameter.class);
+		if (maxCoreParam != null)
+			return maxCoreParam.getValue();
 		return GlobalConstants.maxNumCores;
 	}
 }

From 9fb3b01b69472d3ea1a2b7b3304515131c2306e6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 16 Feb 2015 15:59:41 +0800
Subject: [PATCH 700/881] Plots tuning round time

processTuningRoundTime() have been added and plots tuning round time.
---
 .../mit/streamjit/util/TimeLogProcessor.java  | 46 ++++++++++++++++---
 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index a96f74a6..fb39aa3b 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -108,6 +108,33 @@ private static List<Integer> processDrainTime(String appName)
 		return ret;
 	}
 
+	private static List<Integer> processTuningRoundTime(String appName)
+			throws IOException {
+
+		BufferedReader reader = new BufferedReader(new FileReader(
+				String.format("%s%srunTime.txt", appName, File.separator)));
+		FileWriter writer = new FileWriter(String.format(
+				"%s%sProcessedTuningRoundTime.txt", appName, File.separator));
+		String line;
+		int i = 0;
+		List<Integer> ret = new ArrayList<Integer>(3000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("Tuning")) {
+				String[] arr = line.split(" ");
+				String time = arr[4].trim();
+				time = time.substring(0, time.length() - 1);
+				int val = Integer.parseInt(time);
+				ret.add(val);
+				String data = String.format("%-6d\t%-6d\n", ++i, val);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+		return ret;
+	}
+
 	private static void writeHeapStat(String fileName) throws IOException {
 		List<Integer> heapSize = processSNHeap(fileName, false);
 		List<Integer> heapMaxSize = processSNHeap(fileName, true);
@@ -147,6 +174,7 @@ public static void summarize(String appName) throws IOException {
 		List<Integer> compileTime = processCompileTime(appName);
 		List<Integer> runTime = processRunTime(appName);
 		List<Integer> drainTime = processDrainTime(appName);
+		List<Integer> tuningRoundTime = processTuningRoundTime(appName);
 		String dataFile = "totalStats.txt";
 
 		// String summaryDir = String.format("%s%ssummary", appName,
@@ -159,10 +187,11 @@ public static void summarize(String appName) throws IOException {
 				File.separator, dataFile));
 		int min = Integer.MAX_VALUE;
 
-		for (int i = 0; i < runTime.size(); i++) {
+		for (int i = 0; i < tuningRoundTime.size(); i++) {
 			min = Math.min(min, runTime.get(i));
-			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n", i + 1,
-					compileTime.get(i), runTime.get(i), drainTime.get(i), min);
+			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n",
+					i + 1, tuningRoundTime.get(i), compileTime.get(i),
+					runTime.get(i), drainTime.get(i), min);
 
 			writer.write(msg);
 		}
@@ -187,16 +216,19 @@ private static void makePlotFile(File dir, String name, String dataFile)
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
-				.format("plot \"%s\" using 1:5 with linespoints title \"Current best running time\"\n",
+				.format("plot \"%s\" using 1:6 with linespoints title \"Current best running time\"\n",
 						dataFile));
 		writer.write(String
-				.format("plot \"%s\" using 1:2 with linespoints title \"Compile time\"\n",
+				.format("plot \"%s\" using 1:3 with linespoints title \"Compile time\"\n",
 						dataFile));
 		writer.write(String.format(
-				"plot \"%s\" using 1:3 with linespoints title \"Run time\"\n",
+				"plot \"%s\" using 1:4 with linespoints title \"Run time\"\n",
 				dataFile));
 		writer.write(String
-				.format("plot \"%s\" using 1:4 with linespoints title \"Drain time\"\n",
+				.format("plot \"%s\" using 1:5 with linespoints title \"Drain time\"\n",
+						dataFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:2 with linespoints title \"Tuning Round time\"\n",
 						dataFile));
 		writer.close();
 	}

From 10009a2f4eaa03b577072c3bb0f001e10cfa8133 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 16 Feb 2015 17:46:21 +0800
Subject: [PATCH 701/881] Utils.generateBlobGraphs() added.

---
 .../impl/distributed/common/Utils.java        | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index c4365e54..766417e4 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -14,10 +14,19 @@
 
 import com.google.common.collect.ImmutableSet;
 
+import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
+import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.IOInfo;
+import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.HotSpotTuning;
+import edu.mit.streamjit.impl.distributed.PartitionManager;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
+import edu.mit.streamjit.test.apps.fmradio.FMRadio;
+import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
  * @author Sumanan sumanan@mit.edu
@@ -185,4 +194,44 @@ public static FileWriter fileWriter(String name, boolean append) {
 		}
 		return fw;
 	}
+
+	/**
+	 * [16-02-2015] - I couldn't run dot tools in Lanka cluster. So as a hack, i
+	 * implemented this method to generate blob graph for each configuration.
+	 * TODO: This generation process is damn slow. Takes 40 mins to process 5000
+	 * cfgs.
+	 * 
+	 * @param stream
+	 * @throws IOException
+	 */
+	public static void generateBlobGraphs(OneToOneElement<?, ?> stream)
+			throws IOException {
+		StreamJitApp<?, ?> app = new StreamJitApp<>(stream);
+		PartitionManager partitionManager = new HotSpotTuning(app);
+		partitionManager.getDefaultConfiguration(
+				Workers.getAllWorkersInGraph(app.source), 2);
+		ConfigurationManager cfgManager = new ConfigurationManager(app,
+				partitionManager);
+
+		for (Integer i = 1; i < -5010; i++) {
+			String prefix = i.toString();
+			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
+					prefix);
+			if (cfg != null) {
+				cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+				cfgManager.newConfiguration(cfg);
+			}
+		}
+
+		Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
+				"final");
+		if (cfg != null) {
+			cfg = ConfigurationUtils.addConfigPrefix(cfg, "final");
+			cfgManager.newConfiguration(cfg);
+		}
+	}
+
+	public static void main(String[] args) throws IOException {
+		generateBlobGraphs(new FMRadio.FMRadioCore());
+	}
 }

From 4500ad0ab584cf58ef5a53dfb2d1fb25de0f3a4a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 16 Feb 2015 23:54:21 +0800
Subject: [PATCH 702/881] Utils.generateBlobGraphs prints execution time.

---
 src/edu/mit/streamjit/impl/distributed/common/Utils.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 766417e4..247ae817 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -11,7 +11,9 @@
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableSet;
 
 import edu.mit.streamjit.api.OneToOneElement;
@@ -212,8 +214,8 @@ public static void generateBlobGraphs(OneToOneElement<?, ?> stream)
 				Workers.getAllWorkersInGraph(app.source), 2);
 		ConfigurationManager cfgManager = new ConfigurationManager(app,
 				partitionManager);
-
-		for (Integer i = 1; i < -5010; i++) {
+		Stopwatch sw = Stopwatch.createStarted();
+		for (Integer i = 1; i < 5010; i++) {
 			String prefix = i.toString();
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
 					prefix);
@@ -229,6 +231,8 @@ public static void generateBlobGraphs(OneToOneElement<?, ?> stream)
 			cfg = ConfigurationUtils.addConfigPrefix(cfg, "final");
 			cfgManager.newConfiguration(cfg);
 		}
+		sw.stop();
+		System.out.println(sw.elapsed(TimeUnit.SECONDS));
 	}
 
 	public static void main(String[] args) throws IOException {

From 69a6ff2b88563193c1211e56bfe59d7989db3203 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 09:38:52 +0800
Subject: [PATCH 703/881] Stopped waiting for dot process.

Commented p.waitFor() for some performance improvement.
---
 src/edu/mit/streamjit/impl/distributed/Visualizer.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 8862f9f0..26b257f0 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -221,8 +221,10 @@ private void runDot(String file) {
 					"-o", outFileName);
 			try {
 				Process p = pb.start();
-				p.waitFor();
-			} catch (IOException | InterruptedException e) {
+				// TODO: [20-2-2015]. I am commenting the following line for
+				// some performance improvement. Look for bugs.
+				// p.waitFor();
+			} catch (IOException e) {
 				System.err
 						.println("DotVisualizer: dot(Graphviz) tool is not properly installed in the system");
 				hasDot = false;

From c20b0d24d2f7ed5f87f66cce472f8f5bde740ffd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 11:00:10 +0800
Subject: [PATCH 704/881] A field has been made final.

---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index b4158bf4..b1fb9343 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -361,7 +361,7 @@ public static final class BlockingTailChannel2
 
 		private boolean skipLatchUp;
 
-		private Stopwatch stopWatch;
+		private final Stopwatch stopWatch;
 
 		/**
 		 * @param buffer

From d91013f7b6029e4b5fd13c5afd7e795a3cf8e3bb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 14:14:09 +0800
Subject: [PATCH 705/881] Reads configurations from a file and run it.

verifyTuningTimes() reads configuration prefixes from a file and run those
configurations.
---
 .../distributed/runtimer/OnlineTuner.java     | 34 +++++++++++++++----
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 47a0c794..2271bae0 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -1,8 +1,12 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -55,13 +59,11 @@ public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
 		else if (GlobalConstants.tune == 2)
-			// verifyTuningTimes();
-			evaluate();
+			verifyTuningTimes(cfgPrefixes());
 		else
 			System.err
 					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
 	}
-
 	private void tune() {
 		int round = 0;
 		// Keeps track of the current best time. Uses this to discard bad cfgs
@@ -173,10 +175,7 @@ private void evaluate() {
 	 * 
 	 * This method can be called after the completion of the tuning.
 	 */
-	private void verifyTuningTimes() {
-		String[] cfgPrefixes = { "10", "50", "100", "150", "200", "250", "300",
-				"350", "400", "450", "500", "550", "600", "650", "700", "750",
-				"800", "850", "900", "950", "1000", "hand", "final" };
+	private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
 		for (String prefix : cfgPrefixes) {
 			String cfgName = String.format("%s_%s.cfg", prefix, app.name);
 			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
@@ -353,4 +352,25 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			e1.printStackTrace();
 		}
 	}
+
+	private Iterable<String> cfgPrefixes() {
+		List<String> cfgPrefixes = new ArrayList<String>();
+		cfgPrefixes.add("final");
+		cfgPrefixes.add("hand");
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					String.format("%s%sverify.txt", app.name, File.separator)));
+			String line;
+			while ((line = reader.readLine()) != null) {
+				String[] arr = line.split(",");
+				for (String s : arr) {
+					cfgPrefixes.add(s.trim());
+				}
+			}
+			reader.close();
+		} catch (IOException e) {
+			// e.printStackTrace();
+		}
+		return cfgPrefixes;
+	}
 }
\ No newline at end of file

From 4c716a1f5c366c47e6365e3ef1a39a06e164b942 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 14:19:16 +0800
Subject: [PATCH 706/881] verifyTuningTimes() writes Draindata statistics

---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2271bae0..d206f42c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -187,6 +187,12 @@ private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
 			cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
 			evaluateConfig(cfg, cfgName);
 		}
+
+		try {
+			drainer.dumpDraindataStatistics();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
 		terminate();
 	}
 

From 7c38c13b6221de9de5e3112bbc851688bc4654ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 14:21:14 +0800
Subject: [PATCH 707/881] Obsolete method evaluate() has been removed.

verifyTuningTimes() covers evaluate() method's functionalities.
---
 .../distributed/runtimer/OnlineTuner.java     | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d206f42c..efd9c15b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -152,23 +152,6 @@ private void startTuner() throws IOException {
 		tuner.writeLine(Jsonifiers.toJson(app.getConfiguration()).toString());
 	}
 
-	private void evaluate() {
-		Configuration finalCfg = ConfigurationUtils.readConfiguration(app.name,
-				"final");
-		evaluateConfig(finalCfg, "Final configuration");
-
-		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
-				"hand");
-		evaluateConfig(handCfg, "Handtuned configuration");
-
-		try {
-			drainer.dumpDraindataStatistics();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		terminate();
-	}
-
 	/**
 	 * This method just picks a few configurations and re-run the app to ensure
 	 * the time we reported to the opentuner is correct.
@@ -327,7 +310,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		System.out.println("Evaluating " + cfgName);
 		FileWriter writer;
 		double total = 0;
-		int count = 4;
+		int count = 2;
 		try {
 			writer = new FileWriter(String.format("%s%sEval_%s.txt", app.name,
 					File.separator, app.name), true);

From ab11afadbf6a4722ffe35c314fc73a773c2b383b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 15:02:17 +0800
Subject: [PATCH 708/881] TimeLogger.newConfiguration() takes cfgPrfx as arg

---
 src/edu/mit/streamjit/impl/common/TimeLogger.java   |  6 +++++-
 .../mit/streamjit/impl/distributed/TimeLoggers.java | 13 ++++++++-----
 .../impl/distributed/runtimer/OnlineTuner.java      | 13 ++++++-------
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index 997e3125..23d5479b 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -128,6 +128,10 @@ public interface TimeLogger {
 	 * This method shall be called to indicate the logger that a new
 	 * configuration has been received. Appropriate caller would be
 	 * {@link OnlineTuner}.
+	 * 
+	 * @param cfgPrefix
+	 *            The prefix name of the new {@link Configuration}. Pass null or
+	 *            empty string if the prefix is unknown.
 	 */
-	public void newConfiguration();
+	public void newConfiguration(String cfgPrefix);
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 4f7c4337..4cab4c3d 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -89,7 +89,7 @@ public void logRunTime(String msg) {
 		}
 
 		@Override
-		public void newConfiguration() {
+		public void newConfiguration(String cfgPrefix) {
 		}
 
 		@Override
@@ -225,13 +225,16 @@ public void logRunTime(String msg) {
 		}
 
 		@Override
-		public void newConfiguration() {
-			updateTuningRoundTime();
+		public void newConfiguration(String cfgPrefix) {
 			reconfigNo++;
+			if (cfgPrefix == null || cfgPrefix.isEmpty())
+				cfgPrefix = new Integer(reconfigNo).toString();
+
+			updateTuningRoundTime();
 
 			String msg = String
-					.format("----------------------------%d----------------------------\n",
-							reconfigNo);
+					.format("----------------------------%s----------------------------\n",
+							cfgPrefix);
 			write(compileTimeWriter, msg);
 			write(runTimeWriter, msg);
 			write(drainTimeWriter, msg);
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index efd9c15b..64f0be5c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -76,7 +76,6 @@ private void tune() {
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
-				logger.newConfiguration();
 				if (cfgJson == null) {
 					System.err.println("OpenTuner closed unexpectly.");
 					break;
@@ -89,16 +88,16 @@ private void tune() {
 					break;
 				}
 
+				String cfgPrefix = new Integer(++round).toString();
 				System.out.println(String.format(
-						"---------------------%d-------------------------",
-						++round));
+						"---------------------%s-------------------------",
+						cfgPrefix));
+				logger.newConfiguration(cfgPrefix);
 				Configuration config = Configuration.fromJson(cfgJson);
-				config = ConfigurationUtils.addConfigPrefix(config,
-						new Integer(round).toString());
+				config = ConfigurationUtils.addConfigPrefix(config, cfgPrefix);
 
 				if (GlobalConstants.saveAllConfigurations)
-					ConfigurationUtils.saveConfg(cfgJson,
-							new Integer(round).toString(), app.name);
+					ConfigurationUtils.saveConfg(cfgJson, cfgPrefix, app.name);
 
 				ret = reconfigure(config, 2 * currentBestTime);
 				if (ret.first) {

From 3bbbc61c28a3d0bb8018403948bce12d9433368f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 15:10:52 +0800
Subject: [PATCH 709/881] OnlineTuner.tune() refactored and newcfg() created

The codes, which handle initial operations when a new configuration is received,
have been moved to a new method newCfg().
---
 .../distributed/runtimer/OnlineTuner.java     | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 64f0be5c..b0edd9ce 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -64,6 +64,7 @@ else if (GlobalConstants.tune == 2)
 			System.err
 					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
 	}
+
 	private void tune() {
 		int round = 0;
 		// Keeps track of the current best time. Uses this to discard bad cfgs
@@ -88,17 +89,7 @@ private void tune() {
 					break;
 				}
 
-				String cfgPrefix = new Integer(++round).toString();
-				System.out.println(String.format(
-						"---------------------%s-------------------------",
-						cfgPrefix));
-				logger.newConfiguration(cfgPrefix);
-				Configuration config = Configuration.fromJson(cfgJson);
-				config = ConfigurationUtils.addConfigPrefix(config, cfgPrefix);
-
-				if (GlobalConstants.saveAllConfigurations)
-					ConfigurationUtils.saveConfg(cfgJson, cfgPrefix, app.name);
-
+				Configuration config = newCfg(++round, cfgJson);
 				ret = reconfigure(config, 2 * currentBestTime);
 				if (ret.first) {
 					long time = ret.second;
@@ -288,6 +279,19 @@ private void handleTermination() throws IOException {
 		}
 	}
 
+	private Configuration newCfg(int round, String cfgJson) {
+		String cfgPrefix = new Integer(round).toString();
+		System.out.println(String.format(
+				"---------------------%s-------------------------", cfgPrefix));
+		logger.newConfiguration(cfgPrefix);
+		Configuration config = Configuration.fromJson(cfgJson);
+		config = ConfigurationUtils.addConfigPrefix(config, cfgPrefix);
+
+		if (GlobalConstants.saveAllConfigurations)
+			ConfigurationUtils.saveConfg(cfgJson, cfgPrefix, app.name);
+		return config;
+	}
+
 	private void terminate() {
 		if (manager.isRunning()) {
 			// drainer.startDraining(1);

From 47cfc9bd0411feb969980be3c3a7ba848c79fea7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 15:16:26 +0800
Subject: [PATCH 710/881] OnlineTuner.tune() refactored

A new method tuningFinished() has been added.
---
 .../distributed/runtimer/OnlineTuner.java     | 32 +++++++++++--------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b0edd9ce..320b9d80 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -108,20 +108,7 @@ private void tune() {
 			terminate();
 		}
 
-		try {
-			drainer.dumpDraindataStatistics();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-
-		if (needTermination)
-			terminate();
-
-		try {
-			TimeLogProcessor.summarize(app.name);
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
+		tuningFinished();
 	}
 
 	private void startTuner() throws IOException {
@@ -292,6 +279,23 @@ private Configuration newCfg(int round, String cfgJson) {
 		return config;
 	}
 
+	private void tuningFinished() {
+		try {
+			drainer.dumpDraindataStatistics();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		if (needTermination)
+			terminate();
+
+		try {
+			TimeLogProcessor.summarize(app.name);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
 	private void terminate() {
 		if (manager.isRunning()) {
 			// drainer.startDraining(1);

From 0e2b4fd6840d58bed141b516402871eda7bd5656 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 17:05:09 +0800
Subject: [PATCH 711/881] GlobalConstants.storeProperties() added.

---
 .../distributed/common/GlobalConstants.java   | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 604884ed..01de38aa 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -1,5 +1,10 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Properties;
+
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.TailChannels;
@@ -152,4 +157,44 @@ private GlobalConstants() {
 	 * Turn On/Off the profiling.
 	 */
 	public static final boolean needProfiler = true;
+
+	public static void storeProperties() {
+		Properties prop = new Properties();
+		OutputStream output = null;
+		try {
+			output = new FileOutputStream("globalConstants.properties");
+			prop.setProperty("tunerStartMode",
+					new Integer(tunerStartMode).toString());
+			prop.setProperty("useDrainData",
+					new Boolean(useDrainData).toString());
+			prop.setProperty("needDrainDeadlockHandler", new Boolean(
+					needDrainDeadlockHandler).toString());
+			prop.setProperty("tune", new Integer(tune).toString());
+			prop.setProperty("saveAllConfigurations", new Boolean(
+					saveAllConfigurations).toString());
+			prop.setProperty("outputCount", new Integer(outputCount).toString());
+			prop.setProperty("useCompilerBlob",
+					new Boolean(useCompilerBlob).toString());
+			prop.setProperty("printOutputCountPeriod", new Integer(
+					printOutputCountPeriod).toString());
+			prop.setProperty("singleNodeOnline",
+					new Boolean(singleNodeOnline).toString());
+			prop.setProperty("maxNumCores", new Integer(maxNumCores).toString());
+			prop.setProperty("needProfiler",
+					new Boolean(needProfiler).toString());
+
+			prop.store(output, null);
+
+		} catch (IOException io) {
+			io.printStackTrace();
+		} finally {
+			if (output != null) {
+				try {
+					output.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
 }

From 7bf53374f7dc37c771e5a90990cb484e982fec43 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 20:06:28 +0800
Subject: [PATCH 712/881] storeProperties() refactored, setProperty() added

---
 .../distributed/common/GlobalConstants.java   | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 01de38aa..aca3034e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -163,26 +163,18 @@ public static void storeProperties() {
 		OutputStream output = null;
 		try {
 			output = new FileOutputStream("globalConstants.properties");
-			prop.setProperty("tunerStartMode",
-					new Integer(tunerStartMode).toString());
-			prop.setProperty("useDrainData",
-					new Boolean(useDrainData).toString());
-			prop.setProperty("needDrainDeadlockHandler", new Boolean(
-					needDrainDeadlockHandler).toString());
-			prop.setProperty("tune", new Integer(tune).toString());
-			prop.setProperty("saveAllConfigurations", new Boolean(
-					saveAllConfigurations).toString());
-			prop.setProperty("outputCount", new Integer(outputCount).toString());
-			prop.setProperty("useCompilerBlob",
-					new Boolean(useCompilerBlob).toString());
-			prop.setProperty("printOutputCountPeriod", new Integer(
-					printOutputCountPeriod).toString());
-			prop.setProperty("singleNodeOnline",
-					new Boolean(singleNodeOnline).toString());
-			prop.setProperty("maxNumCores", new Integer(maxNumCores).toString());
-			prop.setProperty("needProfiler",
-					new Boolean(needProfiler).toString());
-
+			setProperty(prop, "tunerStartMode", tunerStartMode);
+			setProperty(prop, "useDrainData", useDrainData);
+			setProperty(prop, "needDrainDeadlockHandler",
+					needDrainDeadlockHandler);
+			setProperty(prop, "tune", tune);
+			setProperty(prop, "saveAllConfigurations", saveAllConfigurations);
+			setProperty(prop, "outputCount", outputCount);
+			setProperty(prop, "useCompilerBlob", useCompilerBlob);
+			setProperty(prop, "printOutputCountPeriod", printOutputCountPeriod);
+			setProperty(prop, "singleNodeOnline", singleNodeOnline);
+			setProperty(prop, "maxNumCores", maxNumCores);
+			setProperty(prop, "needProfiler", needProfiler);
 			prop.store(output, null);
 
 		} catch (IOException io) {
@@ -197,4 +189,12 @@ public static void storeProperties() {
 			}
 		}
 	}
+
+	private static void setProperty(Properties prop, String name, Integer val) {
+		prop.setProperty(name, val.toString());
+	}
+
+	private static void setProperty(Properties prop, String name, Boolean val) {
+		prop.setProperty(name, val.toString());
+	}
 }

From 10e36369bbfc0318a33faaee9ddc36f3283e5518 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 20:33:06 +0800
Subject: [PATCH 713/881] Sets GlobalConstant values from a properties file.

---
 .../distributed/common/GlobalConstants.java   | 58 +++++++++++++++----
 1 file changed, 46 insertions(+), 12 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index aca3034e..eb134e27 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -1,7 +1,9 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Properties;
 
@@ -82,19 +84,19 @@ private GlobalConstants() {
 	 * tuning on remote machines.
 	 * </ol>
 	 */
-	public static final int tunerStartMode = 0;
+	public static final int tunerStartMode;
 
 	/**
 	 * To turn on or turn off the drain data. If this is false, drain data will
 	 * be ignored and every new reconfiguration will run with fresh inputs.
 	 */
-	public static final boolean useDrainData = false;
+	public static final boolean useDrainData;
 
 	/**
 	 * To turn on or off the dead lock handler. see {@link AbstractDrainer} for
 	 * it's usage.
 	 */
-	public static final boolean needDrainDeadlockHandler = true;
+	public static final boolean needDrainDeadlockHandler;
 
 	/**
 	 * Enables tuning. Tuner will be started iff this flag is set true.
@@ -111,37 +113,37 @@ private GlobalConstants() {
 	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
 	 * cfg. Both file should be presented in the running directory.
 	 */
-	public static final int tune = 1;
+	public static final int tune;
 
 	/**
 	 * Save all configurations tired by open tuner in to
 	 * "configurations//app.name" directory.
 	 */
-	public static final boolean saveAllConfigurations = true;
+	public static final boolean saveAllConfigurations;
 
 	/**
 	 * Output count for tuning. Tuner measures the running time for this number
 	 * of outputs.
 	 */
-	public static final int outputCount = 100000;
+	public static final int outputCount;
 
 	/**
 	 * if true uses Compiler2, interpreter otherwise.
 	 */
-	public static final boolean useCompilerBlob = true;
+	public static final boolean useCompilerBlob;
 
 	/**
 	 * Period to print output count periodically. This printing feature get
 	 * turned off if this value is less than 1. Time unit is ms. See
 	 * {@link TailChannels}.
 	 */
-	public static final int printOutputCountPeriod = 6000;
+	public static final int printOutputCountPeriod;
 
 	/**
 	 * Enables {@link DistributedStreamCompiler} to run on a single node. When
 	 * this is enabled, noOfNodes passed as compiler argument has no effect.
 	 */
-	public static final boolean singleNodeOnline = true;
+	public static final boolean singleNodeOnline;
 
 	/**
 	 * We can set this value at class loading time also as follows.
@@ -151,18 +153,50 @@ private GlobalConstants() {
 	 * 
 	 * Lets hard code this for the moment.
 	 */
-	public static final int maxNumCores = 24;
+	public static final int maxNumCores;
 
 	/**
 	 * Turn On/Off the profiling.
 	 */
-	public static final boolean needProfiler = true;
+	public static final boolean needProfiler;
+
+	static {
+		Properties prop = loadProperties();
+		printOutputCountPeriod = Integer.parseInt(prop
+				.getProperty("printOutputCountPeriod"));;
+		maxNumCores = Integer.parseInt(prop.getProperty("maxNumCores"));
+		useCompilerBlob = Boolean.parseBoolean(prop
+				.getProperty("useCompilerBlob"));
+		needDrainDeadlockHandler = Boolean.parseBoolean(prop
+				.getProperty("needDrainDeadlockHandler"));
+		needProfiler = Boolean.parseBoolean(prop.getProperty("needProfiler"));
+		outputCount = Integer.parseInt(prop.getProperty("outputCount"));
+		tune = Integer.parseInt(prop.getProperty("tune"));
+		tunerStartMode = Integer.parseInt(prop.getProperty("tunerStartMode"));
+		saveAllConfigurations = Boolean.parseBoolean(prop
+				.getProperty("saveAllConfigurations"));
+		singleNodeOnline = Boolean.parseBoolean(prop
+				.getProperty("singleNodeOnline"));
+		useDrainData = Boolean.parseBoolean(prop.getProperty("useDrainData"));
+	}
+
+	private static Properties loadProperties() {
+		Properties prop = new Properties();
+		InputStream input = null;
+		try {
+			input = new FileInputStream("options.properties");
+			prop.load(input);
+		} catch (IOException ex) {
+			System.err.println("Failed to load options.properties");
+		}
+		return prop;
+	}
 
 	public static void storeProperties() {
 		Properties prop = new Properties();
 		OutputStream output = null;
 		try {
-			output = new FileOutputStream("globalConstants.properties");
+			output = new FileOutputStream("options.properties");
 			setProperty(prop, "tunerStartMode", tunerStartMode);
 			setProperty(prop, "useDrainData", useDrainData);
 			setProperty(prop, "needDrainDeadlockHandler",

From 3b120411633c9daa3802ae92706729c345ffd332 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 20:42:05 +0800
Subject: [PATCH 714/881] getProperties() method has been added.

---
 .../distributed/common/GlobalConstants.java   | 31 ++++++++++---------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index eb134e27..064f9fd2 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -193,24 +193,11 @@ private static Properties loadProperties() {
 	}
 
 	public static void storeProperties() {
-		Properties prop = new Properties();
 		OutputStream output = null;
 		try {
 			output = new FileOutputStream("options.properties");
-			setProperty(prop, "tunerStartMode", tunerStartMode);
-			setProperty(prop, "useDrainData", useDrainData);
-			setProperty(prop, "needDrainDeadlockHandler",
-					needDrainDeadlockHandler);
-			setProperty(prop, "tune", tune);
-			setProperty(prop, "saveAllConfigurations", saveAllConfigurations);
-			setProperty(prop, "outputCount", outputCount);
-			setProperty(prop, "useCompilerBlob", useCompilerBlob);
-			setProperty(prop, "printOutputCountPeriod", printOutputCountPeriod);
-			setProperty(prop, "singleNodeOnline", singleNodeOnline);
-			setProperty(prop, "maxNumCores", maxNumCores);
-			setProperty(prop, "needProfiler", needProfiler);
+			Properties prop = getProperties();
 			prop.store(output, null);
-
 		} catch (IOException io) {
 			io.printStackTrace();
 		} finally {
@@ -231,4 +218,20 @@ private static void setProperty(Properties prop, String name, Integer val) {
 	private static void setProperty(Properties prop, String name, Boolean val) {
 		prop.setProperty(name, val.toString());
 	}
+
+	public static Properties getProperties() {
+		Properties prop = new Properties();
+		setProperty(prop, "tunerStartMode", tunerStartMode);
+		setProperty(prop, "useDrainData", useDrainData);
+		setProperty(prop, "needDrainDeadlockHandler", needDrainDeadlockHandler);
+		setProperty(prop, "tune", tune);
+		setProperty(prop, "saveAllConfigurations", saveAllConfigurations);
+		setProperty(prop, "outputCount", outputCount);
+		setProperty(prop, "useCompilerBlob", useCompilerBlob);
+		setProperty(prop, "printOutputCountPeriod", printOutputCountPeriod);
+		setProperty(prop, "singleNodeOnline", singleNodeOnline);
+		setProperty(prop, "maxNumCores", maxNumCores);
+		setProperty(prop, "needProfiler", needProfiler);
+		return prop;
+	}
 }

From a94c128d833ce6b9fee6d67a72714e051b62d707 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 21:12:21 +0800
Subject: [PATCH 715/881] Utils store GlobalConstants.properties.

---
 .../impl/distributed/common/Utils.java        | 26 +++----------------
 1 file changed, 3 insertions(+), 23 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 247ae817..bf06f53e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -10,6 +10,7 @@
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Collections;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -138,35 +139,14 @@ public static void writeReadMeTxt(String appName) {
 			Calendar cal = Calendar.getInstance();
 			writer.write(dateFormat.format(cal.getTime()) + "\n");
 			writer.write(appName + "\n");
-			update(writer, "tunerStartMode", GlobalConstants.tunerStartMode);
-			update(writer, "useDrainData", GlobalConstants.useDrainData);
-			update(writer, "needDrainDeadlockHandler",
-					GlobalConstants.needDrainDeadlockHandler);
-			update(writer, "tune", GlobalConstants.tune);
-			update(writer, "saveAllConfigurations",
-					GlobalConstants.saveAllConfigurations);
-			update(writer, "outputCount", GlobalConstants.outputCount);
-			update(writer, "useCompilerBlob", GlobalConstants.useCompilerBlob);
-			update(writer, "printOutputCountPeriod",
-					GlobalConstants.printOutputCountPeriod);
-			update(writer, "singleNodeOnline", GlobalConstants.singleNodeOnline);
-			update(writer, "maxNumCores", GlobalConstants.maxNumCores);
+			Properties prop = GlobalConstants.getProperties();
+			prop.store(writer, "GlobalConstants.Properties");
 			writer.close();
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
 	}
 
-	private static void update(FileWriter writer, String name, int val)
-			throws IOException {
-		writer.write(String.format("%s=%d\n", name, val));
-	}
-
-	private static void update(FileWriter writer, String name, boolean val)
-			throws IOException {
-		writer.write(String.format("%s=%s\n", name, val ? "True" : "False"));
-	}
-
 	/**
 	 * Creates and returns a {@link FileWriter} with append = false. Suppresses
 	 * {@link IOException} and returns null if exception occurred. This method

From dbd054bb4a6bd5164e870f852f482877109b01c1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 22:32:22 +0800
Subject: [PATCH 716/881] Ant:Sends options.properties to Lanka cluster.

---
 jarapp.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jarapp.xml b/jarapp.xml
index 40f1698b..6e8b85ef 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -87,6 +87,7 @@
 		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
 			<fileset file="StreamNode.jar" />
 			<fileset file="${app}.jar" />
+			<fileset file="options.properties" />
 		</scp>
 	</target>
 

From 50e09ee5229d76fabc4baf800cb37dac217fce9a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Feb 2015 22:36:35 +0800
Subject: [PATCH 717/881] evaluateConfig() updates logger.newConfiguration()

---
 src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 320b9d80..15865b9b 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -326,6 +326,7 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 			if (cfg != null) {
 				Pair<Boolean, Long> ret;
 				for (int i = 0; i < count; i++) {
+					logger.newConfiguration(cfgName);
 					ret = reconfigure(cfg, 0);
 					if (ret.first) {
 						prognosticator.time(ret.second);

From a1efaa09afd7d872d234561cba88823afe044f4a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 21 Feb 2015 22:33:12 +0800
Subject: [PATCH 718/881] Shell script to setup an experiment in a cluster

---
 scripts/setup.sh | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 scripts/setup.sh

diff --git a/scripts/setup.sh b/scripts/setup.sh
new file mode 100644
index 00000000..f3d06ee3
--- /dev/null
+++ b/scripts/setup.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#Author - Sumanan
+#Feb 9, 2015
+#Setup directories and scripts to run a distributed StreamJit app.
+function writeRun(){
+	runfile="run.sh"
+	echo "#!/bin/bash" > $runfile
+	echo "#SBATCH --tasks-per-node=1" >> $runfile
+	echo "#SBATCH -N 1"  >> $runfile
+	echo "#SBATCH --cpu_bind=verbose,cores" >> $runfile
+	echo "#SBATCH --exclusive" >> $runfile
+	echo "cd /data/scratch/sumanan/"$1 >> $runfile
+	echo "mkdir -p $2" >> $runfile
+	echo "cd $2" >> $runfile
+	echo "srun python ../lib/opentuner/streamjit/streamjit2.py 12563 &" >> $runfile
+	echo "cd .." >> $runfile
+	echo "srun -l ../bin/java/jdk1.8.0_25/bin/java -Xmx2048m -jar $1.jar $3" >> $runfile
+}
+
+function writeSN(){
+	runfile="streamnode.sh"
+	echo "#!/bin/bash" > $runfile
+	echo "#SBATCH --tasks-per-node=1" >> $runfile
+	echo "#SBATCH -N $2"  >> $runfile
+	echo "#SBATCH --cpu_bind=verbose,cores" >> $runfile
+	echo "#SBATCH --exclusive" >> $runfile
+	echo "cd /data/scratch/sumanan/"$1 >> $runfile
+	echo "srun --exclusive  --nodes=$2 ../bin/java/jdk1.8.0_25/bin/java -Xmx2048m -jar StreamNode.jar 128.30.116." >> $runfile
+}
+
+function creatdirs(){
+	mkdir -p $1
+	ln -s /data/scratch/sumanan/data $1/data
+	ln -s /data/scratch/sumanan/lib $1/lib
+	cd $1
+}
+
+
+if [ "$#" -ne 3 ]; then
+	echo "Illegal number of parameters"
+	echo "3 arguments must be passed"
+	echo "setup.sh <app> <mainClass> <noOfnodes>"
+	exit
+fi
+
+args=("$@")
+app=${args[0]}
+mainClass=${args[1]}
+nodes=${args[2]}
+totalNodes=$((nodes + 1))
+cd /data/scratch/sumanan
+creatdirs $app
+writeRun $app $mainClass $totalNodes
+writeSN $app $nodes

From 5303b47c313479cd36e57d8c0727df818e0dc9d4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 21 Feb 2015 22:40:01 +0800
Subject: [PATCH 719/881] generates run.sh based on tune arg value.

---
 scripts/setup.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scripts/setup.sh b/scripts/setup.sh
index f3d06ee3..4db01f99 100644
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -4,16 +4,19 @@
 #Setup directories and scripts to run a distributed StreamJit app.
 function writeRun(){
 	runfile="run.sh"
+	res=$(get_prop "./$1/options.properties" "tune")
 	echo "#!/bin/bash" > $runfile
 	echo "#SBATCH --tasks-per-node=1" >> $runfile
 	echo "#SBATCH -N 1"  >> $runfile
 	echo "#SBATCH --cpu_bind=verbose,cores" >> $runfile
 	echo "#SBATCH --exclusive" >> $runfile
 	echo "cd /data/scratch/sumanan/"$1 >> $runfile
-	echo "mkdir -p $2" >> $runfile
-	echo "cd $2" >> $runfile
-	echo "srun python ../lib/opentuner/streamjit/streamjit2.py 12563 &" >> $runfile
-	echo "cd .." >> $runfile
+	if [ "$res" -eq "1" ];then
+		echo "mkdir -p $2" >> $runfile
+		echo "cd $2" >> $runfile
+		echo "srun python ../lib/opentuner/streamjit/streamjit2.py 12563 &" >> $runfile
+		echo "cd .." >> $runfile
+	fi
 	echo "srun -l ../bin/java/jdk1.8.0_25/bin/java -Xmx2048m -jar $1.jar $3" >> $runfile
 }
 
@@ -35,6 +38,9 @@ function creatdirs(){
 	cd $1
 }
 
+get_prop(){
+	grep  "^${2}=" ${1}| sed "s%${2}=\(.*\)%\1%"
+}
 
 if [ "$#" -ne 3 ]; then
 	echo "Illegal number of parameters"

From 86fdd5e5aa304a2c12635cdbf236ffdd632189be Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 22 Feb 2015 00:45:16 +0800
Subject: [PATCH 720/881] File path correction in setup.sh.

---
 scripts/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/setup.sh b/scripts/setup.sh
index 4db01f99..3e55bd2a 100644
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -4,7 +4,7 @@
 #Setup directories and scripts to run a distributed StreamJit app.
 function writeRun(){
 	runfile="run.sh"
-	res=$(get_prop "./$1/options.properties" "tune")
+	res=$(get_prop "./options.properties" "tune")
 	echo "#!/bin/bash" > $runfile
 	echo "#SBATCH --tasks-per-node=1" >> $runfile
 	echo "#SBATCH -N 1"  >> $runfile

From 6a1c9dfac6bd1a0c557e0816c7c634a202e660f5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 22 Feb 2015 09:21:00 +0800
Subject: [PATCH 721/881] Comment added to setup.sh script.

---
 scripts/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/setup.sh b/scripts/setup.sh
index 3e55bd2a..19eb00b6 100644
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -55,6 +55,6 @@ mainClass=${args[1]}
 nodes=${args[2]}
 totalNodes=$((nodes + 1))
 cd /data/scratch/sumanan
-creatdirs $app
+creatdirs $app			#Changes the current working directory(CWD).
 writeRun $app $mainClass $totalNodes
 writeSN $app $nodes

From 516975fb00da149feba0886adb08f4b6e194e6b9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 22 Feb 2015 09:26:39 +0800
Subject: [PATCH 722/881] Ant script sends optionsLanka.properties to cluster

Ant script sends optionsLanka.properties to cluster and setup script renames
optionsLanka.properties to options.properties.
---
 jarapp.xml       | 2 +-
 scripts/setup.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index 6e8b85ef..14bb2c26 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -87,7 +87,7 @@
 		<scp trust="true" sftp="true" todir="${username}:${password}@lanka.csail.mit.edu:/data/scratch/${username}/${app}">
 			<fileset file="StreamNode.jar" />
 			<fileset file="${app}.jar" />
-			<fileset file="options.properties" />
+			<fileset file="optionsLanka.properties" />
 		</scp>
 	</target>
 
diff --git a/scripts/setup.sh b/scripts/setup.sh
index 19eb00b6..07aeb15a 100644
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -56,5 +56,6 @@ nodes=${args[2]}
 totalNodes=$((nodes + 1))
 cd /data/scratch/sumanan
 creatdirs $app			#Changes the current working directory(CWD).
+mv "optionsLanka.properties" "options.properties"
 writeRun $app $mainClass $totalNodes
 writeSN $app $nodes

From 8dac27e68040ede3db7bca4d55d72ef10b41f628 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 07:10:43 +0800
Subject: [PATCH 723/881] Updates drainTime.txt with previousCfgPrefix.

When a new configuration come from the OpenTuner, we drain previous
configuration. So the drainTime file should be updated with previous
configuration prefix.
---
 .../mit/streamjit/impl/distributed/TimeLoggers.java | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 4cab4c3d..787075c7 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -224,6 +224,13 @@ public void logRunTime(String msg) {
 			write(runTimeWriter, msg);
 		}
 
+		/**
+		 * [24-02-2015] When a new configuration come from the OpenTuner, we
+		 * drain previous configuration. So the drainTime file should be updated
+		 * with previous configuration prefix.
+		 */
+		String prevcfgPrefix = "";
+
 		@Override
 		public void newConfiguration(String cfgPrefix) {
 			reconfigNo++;
@@ -235,9 +242,13 @@ public void newConfiguration(String cfgPrefix) {
 			String msg = String
 					.format("----------------------------%s----------------------------\n",
 							cfgPrefix);
+			String msg1 = String
+					.format("----------------------------%s----------------------------\n",
+							prevcfgPrefix);
 			write(compileTimeWriter, msg);
 			write(runTimeWriter, msg);
-			write(drainTimeWriter, msg);
+			write(drainTimeWriter, msg1);
+			prevcfgPrefix = cfgPrefix;
 		}
 
 		private void updateTuningRoundTime() {

From 751b9096c0224f1f92d307c4288aec04b521e423 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 07:13:12 +0800
Subject: [PATCH 724/881] AsyncTCPConnection:Stopped printing "SoftClose"

---
 .../streamjit/impl/distributed/common/AsyncTCPConnection.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
index a1235bc8..bf3c569c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/AsyncTCPConnection.java
@@ -176,7 +176,7 @@ public void softClose() throws IOException {
 		this.ooStream.flush();
 		bBAos.writeCompleted();
 		send();
-		System.err.println("Softclose is called");
+		// System.err.println("Softclose is called");
 	}
 
 	/**

From 1706b37ec0d700384fb8bf4338dee25c777a8110 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 07:15:29 +0800
Subject: [PATCH 725/881] Uses AsyncTCPNoParams as the default ConManger.

---
 .../streamjit/impl/distributed/DistributedStreamCompiler.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 84214e1d..36a13fe7 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -119,7 +119,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		PartitionManager partitionManager = new HotSpotTuning(app);
 		ConfigurationManager cfgManager = new ConfigurationManager(app,
 				partitionManager);
-		ConnectionManager conManager = new ConnectionManager.BlockingTCPNoParams(
+		ConnectionManager conManager = new ConnectionManager.AsyncTCPNoParams(
 				controller.controllerNodeID);
 		setConfiguration(controller, app, partitionManager, conManager,
 				cfgManager);

From 0ca1bd82aacd8e354cbd8352ecd8add51a8b6ccf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 11:19:20 +0800
Subject: [PATCH 726/881] BlockingInputChnl:Discards the data if stopType==3

This is added for faster draining. Otherwise it takes fixed 2000ms to drain.
---
 .../impl/distributed/node/BlockingInputChannel.java          | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 5488d496..37528ec9 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -179,7 +179,10 @@ public void receiveData() {
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
-				if (stopType.get() > 0 && ++bufFullCount > 20) {
+				if (stopType.get() == 3) {
+					System.err.println(name + " receiveData:DISCARDING....");
+					break;
+				} else if (stopType.get() > 0 && ++bufFullCount > 20) {
 					this.extraBuffer = new ExtraBuffer();
 					extraBuffer.write(obj);
 					System.err

From e5d43653c2634feb599f4ab505ffa7be471261f1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 15:14:26 +0800
Subject: [PATCH 727/881] getBlobs() refactored and blobSet() method added.

---
 .../node/ConfigurationProcessorImpl.java      | 60 ++++++++++---------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 279588e8..72c32f69 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -139,34 +139,8 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
+			return blobset(blobSet, blobList, drainData, blobConfigs, source);
 
-			for (BlobSpecifier bs : blobList) {
-				Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
-				ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
-				try {
-					BlobFactory bf = bs.getBlobFactory();
-					int maxCores = bs.getCores();
-					Stopwatch sw = Stopwatch.createStarted();
-					DrainData dd = drainData == null ? null : drainData
-							.subset(workIdentifiers);
-					Blob b = bf.makeBlob(workerset, blobConfigs, maxCores, dd);
-					sendCompilationTime(sw, Utils.getblobID(workerset));
-					blobSet.add(b);
-				} catch (Exception ex) {
-					ex.printStackTrace();
-					return null;
-				} catch (OutOfMemoryError er) {
-					Utils.printOutOfMemory();
-					return null;
-				}
-				// DEBUG MSG
-				if (!GlobalConstants.singleNodeOnline)
-					System.out.println(String.format(
-							"A new blob with workers %s has been created.",
-							workIdentifiers.toString()));
-			}
-			System.out.println("All blobs have been created");
-			return blobSet.build();
 		} else
 			return null;
 	}
@@ -285,4 +259,36 @@ private static Class<?> getInngerClass(Class<?> OutterClass,
 						"Innter class %s is not found in the outter class %s. Check the accessibility/visibility of the inner class",
 						InnterClassName, OutterClass.getName()));
 	}
+
+	private ImmutableSet<Blob> blobset(ImmutableSet.Builder<Blob> blobSet,
+			List<BlobSpecifier> blobList, DrainData drainData,
+			Configuration blobConfigs, Worker<?, ?> source) {
+		for (BlobSpecifier bs : blobList) {
+			Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
+			ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
+			try {
+				BlobFactory bf = bs.getBlobFactory();
+				int maxCores = bs.getCores();
+				Stopwatch sw = Stopwatch.createStarted();
+				DrainData dd = drainData == null ? null : drainData
+						.subset(workIdentifiers);
+				Blob b = bf.makeBlob(workerset, blobConfigs, maxCores, dd);
+				sendCompilationTime(sw, Utils.getblobID(workerset));
+				blobSet.add(b);
+			} catch (Exception ex) {
+				ex.printStackTrace();
+				return null;
+			} catch (OutOfMemoryError er) {
+				Utils.printOutOfMemory();
+				return null;
+			}
+			// DEBUG MSG
+			if (!GlobalConstants.singleNodeOnline)
+				System.out.println(String.format(
+						"A new blob with workers %s has been created.",
+						workIdentifiers.toString()));
+		}
+		System.out.println("All blobs have been created");
+		return blobSet.build();
+	}
 }

From ae39740597c8d9d68a5fa82425b4989bb631da09 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 15:24:32 +0800
Subject: [PATCH 728/881] blobSet1() added.It compiles the blobs in parallel

Uses ExecutorService to compiles the blobs in parallel.
---
 .../node/ConfigurationProcessorImpl.java      | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index 72c32f69..fe78c498 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -8,9 +8,15 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
@@ -291,4 +297,87 @@ private ImmutableSet<Blob> blobset(ImmutableSet.Builder<Blob> blobSet,
 		System.out.println("All blobs have been created");
 		return blobSet.build();
 	}
+
+	/**
+	 * Compiles the blobs in parallel.
+	 */
+	private ImmutableSet<Blob> blobset1(ImmutableSet.Builder<Blob> blobSet,
+			List<BlobSpecifier> blobList, DrainData drainData,
+			Configuration blobConfigs, Worker<?, ?> source) {
+		Set<Future<Blob>> futures = new HashSet<>();
+		ExecutorService executerSevce = Executors.newFixedThreadPool(blobList
+				.size());
+
+		for (BlobSpecifier bs : blobList) {
+			MakeBlob mb = new MakeBlob(bs, drainData, blobConfigs, source);
+			Future<Blob> f = executerSevce.submit(mb);
+			futures.add(f);
+		}
+
+		executerSevce.shutdown();
+
+		while (!executerSevce.isTerminated()) {
+			try {
+				Thread.sleep(200);
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+
+		for (Future<Blob> f : futures) {
+			Blob b;
+			try {
+				b = f.get();
+				if (b == null)
+					return null;
+				blobSet.add(b);
+			} catch (InterruptedException | ExecutionException e) {
+				e.printStackTrace();
+			}
+		}
+
+		System.out.println("All blobs have been created");
+		return blobSet.build();
+	}
+
+	private class MakeBlob implements Callable<Blob> {
+		private final BlobSpecifier bs;
+		private final DrainData drainData;
+		private final Configuration blobConfigs;
+		private final Worker<?, ?> source;
+
+		private MakeBlob(BlobSpecifier bs, DrainData drainData,
+				Configuration blobConfigs, Worker<?, ?> source) {
+			this.bs = bs;
+			this.drainData = drainData;
+			this.blobConfigs = blobConfigs;
+			this.source = source;
+		}
+
+		@Override
+		public Blob call() throws Exception {
+			Blob b = null;
+			Set<Integer> workIdentifiers = bs.getWorkerIdentifiers();
+			ImmutableSet<Worker<?, ?>> workerset = bs.getWorkers(source);
+			try {
+				BlobFactory bf = bs.getBlobFactory();
+				int maxCores = bs.getCores();
+				Stopwatch sw = Stopwatch.createStarted();
+				DrainData dd = drainData == null ? null : drainData
+						.subset(workIdentifiers);
+				b = bf.makeBlob(workerset, blobConfigs, maxCores, dd);
+				sendCompilationTime(sw, Utils.getblobID(workerset));
+			} catch (Exception ex) {
+				ex.printStackTrace();
+			} catch (OutOfMemoryError er) {
+				Utils.printOutOfMemory();
+			}
+			// DEBUG MSG
+			if (!GlobalConstants.singleNodeOnline && b != null)
+				System.out.println(String.format(
+						"A new blob with workers %s has been created.",
+						workIdentifiers.toString()));
+			return b;
+		}
+	}
 }

From 5668dabd33ce7117972254c55192849c7ea3ad03 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 15:27:03 +0800
Subject: [PATCH 729/881] Uses blobSet1() to compile the blobs in parallel.

---
 .../impl/distributed/node/ConfigurationProcessorImpl.java    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index fe78c498..adc57bdd 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -145,7 +145,7 @@ private ImmutableSet<Blob> getBlobs(Configuration dyncfg,
 
 			Configuration blobConfigs = dyncfg
 					.getSubconfiguration("blobConfigs");
-			return blobset(blobSet, blobList, drainData, blobConfigs, source);
+			return blobset1(blobSet, blobList, drainData, blobConfigs, source);
 
 		} else
 			return null;
@@ -266,6 +266,9 @@ private static Class<?> getInngerClass(Class<?> OutterClass,
 						InnterClassName, OutterClass.getName()));
 	}
 
+	/**
+	 * Compiles the blobs in serial.
+	 */
 	private ImmutableSet<Blob> blobset(ImmutableSet.Builder<Blob> blobSet,
 			List<BlobSpecifier> blobList, DrainData drainData,
 			Configuration blobConfigs, Worker<?, ?> source) {

From 1a6b01da8d0045b1df9bc0b89eaa5ef35416fc3e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 24 Feb 2015 15:40:46 +0800
Subject: [PATCH 730/881] Noisy debug msgs have been commented.

---
 .../distributed/node/BlockingInputChannel.java |  2 +-
 .../node/ConfigurationProcessorImpl.java       | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
index 37528ec9..fb32bebb 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/BlockingInputChannel.java
@@ -180,7 +180,7 @@ public void receiveData() {
 					e.printStackTrace();
 				}
 				if (stopType.get() == 3) {
-					System.err.println(name + " receiveData:DISCARDING....");
+					// System.err.println(name + " receiveData:DISCARDING....");
 					break;
 				} else if (stopType.get() > 0 && ++bufFullCount > 20) {
 					this.extraBuffer = new ExtraBuffer();
diff --git a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
index adc57bdd..814e83f6 100644
--- a/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
+++ b/src/edu/mit/streamjit/impl/distributed/node/ConfigurationProcessorImpl.java
@@ -291,11 +291,11 @@ private ImmutableSet<Blob> blobset(ImmutableSet.Builder<Blob> blobSet,
 				Utils.printOutOfMemory();
 				return null;
 			}
-			// DEBUG MSG
-			if (!GlobalConstants.singleNodeOnline)
-				System.out.println(String.format(
-						"A new blob with workers %s has been created.",
-						workIdentifiers.toString()));
+			// // DEBUG MSG
+			// if (!GlobalConstants.singleNodeOnline)
+			// System.out.println(String.format(
+			// "A new blob with workers %s has been created.",
+			// workIdentifiers.toString()));
 		}
 		System.out.println("All blobs have been created");
 		return blobSet.build();
@@ -376,10 +376,10 @@ public Blob call() throws Exception {
 				Utils.printOutOfMemory();
 			}
 			// DEBUG MSG
-			if (!GlobalConstants.singleNodeOnline && b != null)
-				System.out.println(String.format(
-						"A new blob with workers %s has been created.",
-						workIdentifiers.toString()));
+			// if (!GlobalConstants.singleNodeOnline && b != null)
+			// System.out.println(String.format(
+			// "A new blob with workers %s has been created.",
+			// workIdentifiers.toString()));
 			return b;
 		}
 	}

From 1e86798c70cea19a247381fdf12c933d2887dfed Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 08:47:52 +0800
Subject: [PATCH 731/881] Stopped storing profile data in a Map.

Profile data goes upto 700MB during a full tuning.
---
 .../impl/distributed/profiler/MasterProfiler.java          | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
index c3d7c93e..6886471e 100644
--- a/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/MasterProfiler.java
@@ -15,7 +15,7 @@
  */
 public class MasterProfiler implements SNProfileElementProcessor {
 
-	private final Map<Integer, SNBufferStatusData> BufferStatusDataMap;
+	// private final Map<Integer, SNBufferStatusData> BufferStatusDataMap;
 
 	private final ProfileElementLogger logger;
 
@@ -24,13 +24,14 @@ public ProfileElementLogger logger() {
 	}
 
 	public MasterProfiler(String appName) {
-		BufferStatusDataMap = new ConcurrentHashMap<>();
+		// BufferStatusDataMap = new ConcurrentHashMap<>();
 		logger = new FileProfileElementLogger(appName);
 	}
 
 	@Override
 	public void process(SNBufferStatusData bufferStatusData) {
-		BufferStatusDataMap.put(bufferStatusData.machineID, bufferStatusData);
+		// BufferStatusDataMap.put(bufferStatusData.machineID,
+		// bufferStatusData);
 		logger.process(bufferStatusData);
 	}
 }

From 5aa80310a263b66fd1025e11efa37137e81cd7b9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 09:44:12 +0800
Subject: [PATCH 732/881] Default constructor added to ChannelVocoder7

---
 .../streamjit/test/apps/channelvocoder7/ChannelVocoder7.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/test/apps/channelvocoder7/ChannelVocoder7.java b/src/edu/mit/streamjit/test/apps/channelvocoder7/ChannelVocoder7.java
index c085fc2a..a8f55b70 100644
--- a/src/edu/mit/streamjit/test/apps/channelvocoder7/ChannelVocoder7.java
+++ b/src/edu/mit/streamjit/test/apps/channelvocoder7/ChannelVocoder7.java
@@ -89,6 +89,10 @@ public ChannelVocoder7Kernel(int numFilters, int numTaps) {
 			add(new LowPassFilter(1, (float) ((2 * Math.PI * 5000) / 8000), 64));
 			add(new MainSplitjoin(numFilters, numTaps));
 		}
+
+		public ChannelVocoder7Kernel() {
+			this(16, 64);
+		}
 	}
 
 	/**

From fcd34894e79e3a03fccdcbe445460a61b3169b1d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 11:09:55 +0800
Subject: [PATCH 733/881] DistAppRunner added.

Runs a benchmark with DistributedStreamCompiler
---
 src/edu/mit/streamjit/test/DistAppRunner.java | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 src/edu/mit/streamjit/test/DistAppRunner.java

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
new file mode 100644
index 00000000..ec34081d
--- /dev/null
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -0,0 +1,45 @@
+package edu.mit.streamjit.test;
+
+import java.io.IOException;
+
+import edu.mit.streamjit.api.CompiledStream;
+import edu.mit.streamjit.api.Input;
+import edu.mit.streamjit.api.Output;
+import edu.mit.streamjit.api.StreamCompiler;
+import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.test.Benchmark.Dataset;
+import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
+import edu.mit.streamjit.test.apps.fmradio.FMRadio.FMRadioBenchmarkProvider;
+import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin.NestedSplitJoinBenchmarkProvider;
+
+public final class DistAppRunner {
+
+	public static void main(String[] args) throws InterruptedException,
+			IOException {
+		int noOfNodes;
+
+		try {
+			noOfNodes = Integer.parseInt(args[0]);
+		} catch (Exception ex) {
+			noOfNodes = 3;
+		}
+
+		Benchmark benchmark = new NestedSplitJoinBenchmarkProvider().iterator()
+				.next();
+		StreamCompiler compiler = new DistributedStreamCompiler(noOfNodes);
+
+		Dataset dataset = benchmark.inputs().get(0);
+		Input<Object> input = Datasets.cycle(dataset.input());
+
+		CompiledStream stream = compiler.compile(benchmark.instantiate(),
+				input, Output.blackHole());
+		stream.awaitDrained();
+	}
+
+	private static void startSNs(int noOfNodes) throws IOException {
+		for (int i = 1; i < noOfNodes; i++)
+			new ProcessBuilder("xterm", "-e", "java", "-jar", "StreamNode.jar")
+					.start();
+		new ProcessBuilder("java", "-jar", "StreamNode.jar").start();
+	}
+}

From be813bed174178c8a03d0c5b88db81cec3027c76 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 11:29:58 +0800
Subject: [PATCH 734/881] DistAppRunner is the new Ant's main class.

---
 jarapp.properties | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/jarapp.properties b/jarapp.properties
index ab081c55..59f0ecf8 100644
--- a/jarapp.properties
+++ b/jarapp.properties
@@ -1,10 +1,8 @@
-#app=NestedSplitJoin
-#mainclasspath=edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin
-#streamGraphName=NestedSplitJoinCore
 app=FMRadio
-mainclasspath=edu.mit.streamjit.test.apps.fmradio.FMRadio
 streamGraphName=FMRadioCore
+#app=NestedSplitJoin
+#streamGraphName=NestedSplitJoinCore
 username=sumanan
 ##password=abc123
+mainclasspath=edu.mit.streamjit.test.DistAppRunner
 numOfSNs=2
-

From 8190bd462ade10f0abe2801420e37e17da6a3c13 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 11:46:06 +0800
Subject: [PATCH 735/881] FMRadio's default constructor's args changed.

Default constructor's argument has been changed to match with the
FMRadioBenchmarkProvider's first benchmark.
---
 src/edu/mit/streamjit/test/apps/fmradio/FMRadio.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/test/apps/fmradio/FMRadio.java b/src/edu/mit/streamjit/test/apps/fmradio/FMRadio.java
index 914aebd9..24302f57 100644
--- a/src/edu/mit/streamjit/test/apps/fmradio/FMRadio.java
+++ b/src/edu/mit/streamjit/test/apps/fmradio/FMRadio.java
@@ -227,7 +227,7 @@ public static final class FMRadioCore extends Pipeline<Float, Float> {
 		private static final float high = 1760;
 
 		public FMRadioCore() {
-			this(11, 64);
+			this(7, 128);
 		}
 
 		public FMRadioCore(int bands, int taps) {

From 2a066c4090b8e9d8ae685f1acb030c72365aac9a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 11:57:03 +0800
Subject: [PATCH 736/881] DistAppRunner uses FMRadio as the default app.

---
 src/edu/mit/streamjit/test/DistAppRunner.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
index ec34081d..24ba0202 100644
--- a/src/edu/mit/streamjit/test/DistAppRunner.java
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -8,10 +8,12 @@
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.test.Benchmark.Dataset;
-import edu.mit.streamjit.test.apps.channelvocoder7.ChannelVocoder7;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio.FMRadioBenchmarkProvider;
-import edu.mit.streamjit.test.sanity.nestedsplitjoinexample.NestedSplitJoin.NestedSplitJoinBenchmarkProvider;
 
+/**
+ * @author sumanan
+ * @since 25 Feb, 2015
+ */
 public final class DistAppRunner {
 
 	public static void main(String[] args) throws InterruptedException,
@@ -24,10 +26,10 @@ public static void main(String[] args) throws InterruptedException,
 			noOfNodes = 3;
 		}
 
-		Benchmark benchmark = new NestedSplitJoinBenchmarkProvider().iterator()
-				.next();
+		// startSNs(noOfNodes);
 		StreamCompiler compiler = new DistributedStreamCompiler(noOfNodes);
 
+		Benchmark benchmark = new FMRadioBenchmarkProvider().iterator().next();
 		Dataset dataset = benchmark.inputs().get(0);
 		Input<Object> input = Datasets.cycle(dataset.input());
 

From ca89acb4598b9e640867e5015c1658d9ea94b3b1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 12:33:20 +0800
Subject: [PATCH 737/881] String format bug fixed.

---
 .../streamjit/impl/compiler2/DoubleArrayConcreteStorage.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/compiler2/DoubleArrayConcreteStorage.java b/src/edu/mit/streamjit/impl/compiler2/DoubleArrayConcreteStorage.java
index 99f5b7c5..d7784108 100644
--- a/src/edu/mit/streamjit/impl/compiler2/DoubleArrayConcreteStorage.java
+++ b/src/edu/mit/streamjit/impl/compiler2/DoubleArrayConcreteStorage.java
@@ -63,7 +63,7 @@ public Object read(int index) {
 		try {
 			return readHandle.invoke(index);
 		} catch (Throwable ex) {
-			throw new AssertionError(String.format("%s.read(%d, %s)", this, index), ex);
+			throw new AssertionError(String.format("%s.read(%d)", this, index), ex);
 		}
 	}
 

From 4c46cee4210b200592296d60f3f991af573545b8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 12:39:43 +0800
Subject: [PATCH 738/881] AllocationStrategy uses GlobalConstants.maxNumCores

Previously this parameter was hard coded to 8 cores.
---
 src/edu/mit/streamjit/impl/compiler2/Compiler2.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
index 8ede59eb..05cf2f2a 100644
--- a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
+++ b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
@@ -1,7 +1,9 @@
 package edu.mit.streamjit.impl.compiler2;
 
 import com.google.common.base.Function;
+
 import static com.google.common.base.Preconditions.checkState;
+
 import com.google.common.base.Predicate;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.HashBasedTable;
@@ -19,6 +21,7 @@
 import com.google.common.primitives.Primitives;
 import com.google.common.reflect.TypeResolver;
 import com.google.common.reflect.TypeToken;
+
 import edu.mit.streamjit.api.DuplicateSplitter;
 import edu.mit.streamjit.api.IllegalStreamGraphException;
 import edu.mit.streamjit.api.Input;
@@ -47,6 +50,7 @@
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.DrainInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.ReadInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.WriteInstruction;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
@@ -57,6 +61,7 @@
 import edu.mit.streamjit.util.ReflectionUtils;
 import edu.mit.streamjit.util.bytecode.Module;
 import edu.mit.streamjit.util.bytecode.ModuleClassLoader;
+
 import java.io.BufferedWriter;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -94,7 +99,7 @@ public class Compiler2 {
 	public static final RemovalStrategy REMOVAL_STRATEGY = new BitsetRemovalStrategy();
 	public static final FusionStrategy FUSION_STRATEGY = new BitsetFusionStrategy();
 	public static final UnboxingStrategy UNBOXING_STRATEGY = new BitsetUnboxingStrategy();
-	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(8);
+	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(GlobalConstants.maxNumCores);
 	public static final StorageStrategy INTERNAL_STORAGE_STRATEGY = new TuneInternalStorageStrategy();
 	public static final StorageStrategy EXTERNAL_STORAGE_STRATEGY = new TuneExternalStorageStrategy();
 	private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup();

From 0c200ceceefbcedc5a339df58e9b097abf728991 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 21:27:40 +0800
Subject: [PATCH 739/881] Updates the README.txt with the benchmarkName.

---
 src/edu/mit/streamjit/test/DistAppRunner.java | 25 +++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
index 24ba0202..4c78ad79 100644
--- a/src/edu/mit/streamjit/test/DistAppRunner.java
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -1,9 +1,12 @@
 package edu.mit.streamjit.test;
 
+import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 
 import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.Input;
+import edu.mit.streamjit.api.OneToOneElement;
 import edu.mit.streamjit.api.Output;
 import edu.mit.streamjit.api.StreamCompiler;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
@@ -33,9 +36,13 @@ public static void main(String[] args) throws InterruptedException,
 		Dataset dataset = benchmark.inputs().get(0);
 		Input<Object> input = Datasets.cycle(dataset.input());
 
-		CompiledStream stream = compiler.compile(benchmark.instantiate(),
-				input, Output.blackHole());
+		OneToOneElement<Object, Object> streamGraph = benchmark.instantiate();
+		CompiledStream stream = compiler.compile(streamGraph, input,
+				Output.blackHole());
 		stream.awaitDrained();
+
+		String appName = streamGraph.getClass().getSimpleName();
+		updateReadMeTxt(appName, benchmark.toString());
 	}
 
 	private static void startSNs(int noOfNodes) throws IOException {
@@ -44,4 +51,18 @@ private static void startSNs(int noOfNodes) throws IOException {
 					.start();
 		new ProcessBuilder("java", "-jar", "StreamNode.jar").start();
 	}
+
+	/**
+	 * [25 Feb, 2015] TODO: This is a temporary fix to update the benchmark name
+	 * ( that is more descriptive than plain appName) to the README.txt.
+	 * Consider passing the benchmarkName to the
+	 * {@link DistributedStreamCompiler} and let it to update the README.txt.
+	 */
+	public static void updateReadMeTxt(String appName, String benchmarkName)
+			throws IOException {
+		FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
+				appName, File.separator), true);
+		writer.write("benchmarkName=" + benchmarkName);
+		writer.close();
+	}
 }

From 3c782d793b42415aef6e12057441ad8a0b8b04ae Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 21:49:51 +0800
Subject: [PATCH 740/881] DistAppRunner updates total running time to README

---
 src/edu/mit/streamjit/test/DistAppRunner.java | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
index 4c78ad79..2a06b96a 100644
--- a/src/edu/mit/streamjit/test/DistAppRunner.java
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -3,6 +3,9 @@
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.api.CompiledStream;
 import edu.mit.streamjit.api.Input;
@@ -22,7 +25,7 @@ public final class DistAppRunner {
 	public static void main(String[] args) throws InterruptedException,
 			IOException {
 		int noOfNodes;
-
+		Stopwatch sw = Stopwatch.createStarted();
 		try {
 			noOfNodes = Integer.parseInt(args[0]);
 		} catch (Exception ex) {
@@ -41,8 +44,10 @@ public static void main(String[] args) throws InterruptedException,
 				Output.blackHole());
 		stream.awaitDrained();
 
+		sw.stop();
+		long elapsedMills = sw.elapsed(TimeUnit.MILLISECONDS);
 		String appName = streamGraph.getClass().getSimpleName();
-		updateReadMeTxt(appName, benchmark.toString());
+		updateReadMeTxt(appName, benchmark.toString(), elapsedMills);
 	}
 
 	private static void startSNs(int noOfNodes) throws IOException {
@@ -58,11 +63,23 @@ private static void startSNs(int noOfNodes) throws IOException {
 	 * Consider passing the benchmarkName to the
 	 * {@link DistributedStreamCompiler} and let it to update the README.txt.
 	 */
-	public static void updateReadMeTxt(String appName, String benchmarkName)
-			throws IOException {
+	private static void updateReadMeTxt(String appName, String benchmarkName,
+			long elapsedMills) throws IOException {
 		FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
 				appName, File.separator), true);
 		writer.write("benchmarkName=" + benchmarkName);
+		writer.write("TotalRunningTime=" + elapsedTime(elapsedMills));
 		writer.close();
 	}
+
+	private static String elapsedTime(long mills) {
+		String hms = String.format(
+				"%02dH:%02dM:%02dS",
+				TimeUnit.MILLISECONDS.toHours(mills),
+				TimeUnit.MILLISECONDS.toMinutes(mills)
+						% TimeUnit.HOURS.toMinutes(1),
+				TimeUnit.MILLISECONDS.toSeconds(mills)
+						% TimeUnit.MINUTES.toSeconds(1));
+		return hms;
+	}
 }

From e8a74096e41f1fa246131a55d01d81f2a491d061 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 25 Feb 2015 22:25:28 +0800
Subject: [PATCH 741/881] updateReadMeTxt() uses String.Format

---
 src/edu/mit/streamjit/test/DistAppRunner.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
index 2a06b96a..4272dfd7 100644
--- a/src/edu/mit/streamjit/test/DistAppRunner.java
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -67,8 +67,9 @@ private static void updateReadMeTxt(String appName, String benchmarkName,
 			long elapsedMills) throws IOException {
 		FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
 				appName, File.separator), true);
-		writer.write("benchmarkName=" + benchmarkName);
-		writer.write("TotalRunningTime=" + elapsedTime(elapsedMills));
+		writer.write(String.format("benchmarkName=%s\n", benchmarkName));
+		writer.write(String.format("TotalRunningTime=%s\n",
+				elapsedTime(elapsedMills)));
 		writer.close();
 	}
 

From b9cffaf7a0e67df503d69260d564fe8f35c0b2f5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Feb 2015 14:59:20 +0800
Subject: [PATCH 742/881] TimeLogProcessor handles Slurm's sysout prefix.

Sometimes Slurm writes "0: " prefix to every sys out lines. Heap status
processor considers this case.
---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index fb39aa3b..bee0f6a4 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -150,6 +150,7 @@ private static void writeHeapStat(String fileName) throws IOException {
 
 	private static List<Integer> processSNHeap(String fileName,
 			Boolean isHeapMax) throws IOException {
+		String slurmPrefix = "0: ";
 		String heapType = "heapSize";
 		if (isHeapMax)
 			heapType = "heapMaxSize";
@@ -158,6 +159,9 @@ private static List<Integer> processSNHeap(String fileName,
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
 		while ((line = reader.readLine()) != null) {
+			// Slurm adds prefix to every sysout line.
+			if (line.startsWith(slurmPrefix))
+				line = line.substring(slurmPrefix.length());
 			if (line.startsWith(heapType)) {
 				String[] arr = line.split(" ");
 				String time = arr[2].trim();

From 0ec316d6644b5fc078afc296805c99e310afe09a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Feb 2015 18:54:51 +0800
Subject: [PATCH 743/881] Generates all output files to summary directory.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 57 +++++++++++--------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index bee0f6a4..c411afa8 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -24,12 +24,13 @@ public static void main(String[] args) throws IOException {
 		summarize("FMRadioCore");
 	}
 
-	private static List<Integer> processCompileTime(String appName)
+	private static List<Integer> processCompileTime(String appName, File outDir)
 			throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%scompileTime.txt", appName, File.separator)));
-		FileWriter writer = new FileWriter(String.format(
-				"%s%sProcessedCompileTime.txt", appName, File.separator));
+
+		File outFile = new File(outDir, "processedCompileTime.txt");
+		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -50,13 +51,14 @@ private static List<Integer> processCompileTime(String appName)
 		return ret;
 	}
 
-	private static List<Integer> processRunTime(String appName)
+	private static List<Integer> processRunTime(String appName, File outDir)
 			throws IOException {
 
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-		FileWriter writer = new FileWriter(String.format(
-				"%s%sProcessedRunTime.txt", appName, File.separator));
+
+		File outFile = new File(outDir, "processedRunTime.txt");
+		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -82,12 +84,13 @@ private static List<Integer> processRunTime(String appName)
 		return ret;
 	}
 
-	private static List<Integer> processDrainTime(String appName)
+	private static List<Integer> processDrainTime(String appName, File outDir)
 			throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sdrainTime.txt", appName, File.separator)));
-		FileWriter writer = new FileWriter(String.format(
-				"%s%sProcessedDrainTime.txt", appName, File.separator));
+
+		File outFile = new File(outDir, "processedDrainTime.txt");
+		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -108,13 +111,13 @@ private static List<Integer> processDrainTime(String appName)
 		return ret;
 	}
 
-	private static List<Integer> processTuningRoundTime(String appName)
-			throws IOException {
+	private static List<Integer> processTuningRoundTime(String appName,
+			File outDir) throws IOException {
 
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-		FileWriter writer = new FileWriter(String.format(
-				"%s%sProcessedTuningRoundTime.txt", appName, File.separator));
+		File outFile = new File(outDir, "processedTuningRoundTime.txt");
+		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
@@ -135,11 +138,14 @@ private static List<Integer> processTuningRoundTime(String appName)
 		return ret;
 	}
 
-	private static void writeHeapStat(String fileName) throws IOException {
+	private static void writeHeapStat(String fileName, File outDir)
+			throws IOException {
 		List<Integer> heapSize = processSNHeap(fileName, false);
 		List<Integer> heapMaxSize = processSNHeap(fileName, true);
-		FileWriter writer = new FileWriter(String.format("%s_heapStatus.txt",
-				fileName));
+
+		String outFileName = String.format("%s_heapStatus.txt", fileName);
+		File outFile = new File(outDir, outFileName);
+		FileWriter writer = new FileWriter(outFile, false);
 		for (int i = 0; i < heapSize.size(); i++) {
 			String msg = String.format("%-6d\t%-6d\t%-6d\n", i + 1,
 					heapSize.get(i), heapMaxSize.get(i));
@@ -175,20 +181,21 @@ private static List<Integer> processSNHeap(String fileName,
 	}
 
 	public static void summarize(String appName) throws IOException {
-		List<Integer> compileTime = processCompileTime(appName);
-		List<Integer> runTime = processRunTime(appName);
-		List<Integer> drainTime = processDrainTime(appName);
-		List<Integer> tuningRoundTime = processTuningRoundTime(appName);
+		File summaryDir = new File(String.format("%s%ssummary", appName,
+				File.separator));
+		Utils.createDir(summaryDir.getPath());
+		List<Integer> compileTime = processCompileTime(appName, summaryDir);
+		List<Integer> runTime = processRunTime(appName, summaryDir);
+		List<Integer> drainTime = processDrainTime(appName, summaryDir);
+		List<Integer> tuningRoundTime = processTuningRoundTime(appName,
+				summaryDir);
 		String dataFile = "totalStats.txt";
 
 		// String summaryDir = String.format("%s%ssummary", appName,
 		// File.separator);
-		File summaryDir = new File(String.format("%s%ssummary", appName,
-				File.separator));
-		Utils.createDir(summaryDir.getPath());
 
-		FileWriter writer = new FileWriter(String.format("%s%s%s", summaryDir,
-				File.separator, dataFile));
+		File outfile = new File(summaryDir, dataFile);
+		FileWriter writer = new FileWriter(outfile, false);
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 0; i < tuningRoundTime.size(); i++) {

From 27602fd786e8d84aa7732a6d4152a33b0ef5bfc0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 26 Feb 2015 19:30:44 +0800
Subject: [PATCH 744/881] backup.sh added.

---
 scripts/backup.sh | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 scripts/backup.sh

diff --git a/scripts/backup.sh b/scripts/backup.sh
new file mode 100644
index 00000000..98ad53d8
--- /dev/null
+++ b/scripts/backup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#Author - Sumanan
+#Feb 25, 2015
+#Backups tuning output files and directories.
+args=("$@")
+suffix=${args[0]}
+if [ -z $suffix ]
+then
+suffix="Orig"
+fi
+
+if [ -d summary$suffix ]; then
+    echo "summary$suffix exists. No backups. Exiting..."
+    exit
+fi
+
+mv summary summary$suffix
+mv compileTime.txt compileTime$suffix.txt
+mv runTime.txt runTime$suffix.txt
+mv drainTime.txt drainTime$suffix.txt
+mv GraphProperty.txt GraphProperty$suffix.txt
+mv README.txt README$suffix.txt
+

From aacff7559baa5485243b85db5ab6ccb0ff59cad8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 27 Feb 2015 08:11:08 +0800
Subject: [PATCH 745/881] Bug in startSNs() fixed.

---
 src/edu/mit/streamjit/test/DistAppRunner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/test/DistAppRunner.java b/src/edu/mit/streamjit/test/DistAppRunner.java
index 4272dfd7..d8acd181 100644
--- a/src/edu/mit/streamjit/test/DistAppRunner.java
+++ b/src/edu/mit/streamjit/test/DistAppRunner.java
@@ -54,7 +54,7 @@ private static void startSNs(int noOfNodes) throws IOException {
 		for (int i = 1; i < noOfNodes; i++)
 			new ProcessBuilder("xterm", "-e", "java", "-jar", "StreamNode.jar")
 					.start();
-		new ProcessBuilder("java", "-jar", "StreamNode.jar").start();
+		// new ProcessBuilder("java", "-jar", "StreamNode.jar").start();
 	}
 
 	/**

From d03d978e33fb40cc52fc1f57f2d8b99ae83f89cf Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 27 Feb 2015 11:46:43 +0800
Subject: [PATCH 746/881] Default const arg matches with benchmark's arg

FilterBankPipeline's default constructor argument is changed to match with
FilterBankBenchmark's arg.
---
 src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java b/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
index b57bac23..5b0f5bcc 100644
--- a/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
+++ b/src/edu/mit/streamjit/test/apps/filterbank6/FilterBank6.java
@@ -54,7 +54,7 @@ private static Dataset dataset() {
 		@Override
 		@SuppressWarnings("unchecked")
 		public OneToOneElement<Object, Object> instantiate() {
-			return (OneToOneElement)new FilterBankPipeline(8);
+			return (OneToOneElement)new FilterBankPipeline(32);
 		}
 	}
 

From 26f9fafa349815e67a1fa3b532a804c18831d3c3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:18:47 +0800
Subject: [PATCH 747/881] processCompileTime returns map with cfgPrfx as key

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index c411afa8..f9ddcadf 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -8,7 +8,9 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
@@ -24,24 +26,28 @@ public static void main(String[] args) throws IOException {
 		summarize("FMRadioCore");
 	}
 
-	private static List<Integer> processCompileTime(String appName, File outDir)
-			throws IOException {
+	private static Map<String, Integer> processCompileTime(String appName,
+			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%scompileTime.txt", appName, File.separator)));
 
 		File outFile = new File(outDir, "processedCompileTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
+		String cfgPrefix = "1";
 		int i = 0;
-		List<Integer> ret = new ArrayList<Integer>(3000);
+		Map<String, Integer> ret = new HashMap<>(5000);
 		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("----------------------------"))
+				cfgPrefix = cfgString(line);
 			if (line.startsWith("Total")) {
 				String[] arr = line.split(" ");
 				String time = arr[3].trim();
 				time = time.substring(0, time.length() - 2);
 				int val = Integer.parseInt(time);
-				ret.add(val);
-				String data = String.format("%d\t%d\n", ++i, val);
+				ret.put(cfgPrefix, val);
+				String data = String
+						.format("%d\t%s\t%d\n", ++i, cfgPrefix, val);
 				writer.write(data);
 			}
 		}
@@ -51,6 +57,11 @@ private static List<Integer> processCompileTime(String appName, File outDir)
 		return ret;
 	}
 
+	private static String cfgString(String line) {
+		String l = line.replace('-', ' ');
+		return l.trim();
+	}
+
 	private static List<Integer> processRunTime(String appName, File outDir)
 			throws IOException {
 

From 86658b8709ce7d7428d242e67215ed07bdf0498a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:28:58 +0800
Subject: [PATCH 748/881] processCompileTime refactored and process() added

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 64 ++++++++-----------
 1 file changed, 28 insertions(+), 36 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index f9ddcadf..051663c1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -33,25 +33,7 @@ private static Map<String, Integer> processCompileTime(String appName,
 
 		File outFile = new File(outDir, "processedCompileTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		String line;
-		String cfgPrefix = "1";
-		int i = 0;
-		Map<String, Integer> ret = new HashMap<>(5000);
-		while ((line = reader.readLine()) != null) {
-			if (line.startsWith("----------------------------"))
-				cfgPrefix = cfgString(line);
-			if (line.startsWith("Total")) {
-				String[] arr = line.split(" ");
-				String time = arr[3].trim();
-				time = time.substring(0, time.length() - 2);
-				int val = Integer.parseInt(time);
-				ret.put(cfgPrefix, val);
-				String data = String
-						.format("%d\t%s\t%d\n", ++i, cfgPrefix, val);
-				writer.write(data);
-			}
-		}
-		writer.flush();
+		Map<String, Integer> ret = process(reader, writer, "Total");
 		reader.close();
 		writer.close();
 		return ret;
@@ -95,27 +77,13 @@ private static List<Integer> processRunTime(String appName, File outDir)
 		return ret;
 	}
 
-	private static List<Integer> processDrainTime(String appName, File outDir)
-			throws IOException {
+	private static Map<String, Integer> processDrainTime(String appName,
+			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sdrainTime.txt", appName, File.separator)));
-
 		File outFile = new File(outDir, "processedDrainTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		String line;
-		int i = 0;
-		List<Integer> ret = new ArrayList<Integer>(3000);
-		while ((line = reader.readLine()) != null) {
-			if (line.startsWith("Drain time")) {
-				String[] arr = line.split(" ");
-				String time = arr[3].trim();
-				time = time.substring(0, time.length() - 2);
-				int val = Integer.parseInt(time);
-				ret.add(val);
-				String data = String.format("%d\t%d\n", ++i, val);
-				writer.write(data);
-			}
-		}
+		Map<String, Integer> ret = process(reader, writer, "Drain time");
 		writer.flush();
 		reader.close();
 		writer.close();
@@ -149,6 +117,30 @@ private static List<Integer> processTuningRoundTime(String appName,
 		return ret;
 	}
 
+	private static Map<String, Integer> process(BufferedReader reader,
+			FileWriter writer, String startString) throws IOException {
+		String line;
+		String cfgPrefix = "1";
+		int i = 0;
+		Map<String, Integer> ret = new HashMap<>(5000);
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("----------------------------"))
+				cfgPrefix = cfgString(line);
+			if (line.startsWith(startString)) {
+				String[] arr = line.split(" ");
+				String time = arr[3].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				ret.put(cfgPrefix, val);
+				String data = String
+						.format("%d\t%s\t%d\n", ++i, cfgPrefix, val);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		return ret;
+	}
+
 	private static void writeHeapStat(String fileName, File outDir)
 			throws IOException {
 		List<Integer> heapSize = processSNHeap(fileName, false);

From 14325e60463c10840c5483a96c431ca6548d3718 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:33:12 +0800
Subject: [PATCH 749/881] Prints tuning round time

---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 787075c7..4a5bd6a5 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -262,7 +262,7 @@ private void updateTuningRoundTime() {
 				tuningRoundSW.start();
 			}
 			write(runTimeWriter,
-					String.format("Tuning round time - %dS\n", time));
+					String.format("Tuning round time %dS\n", time));
 		}
 
 		private static OutputStreamWriter getOSWriter(OutputStream os) {

From 4015061651497090c678f57d2e93911f11b66bb3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:39:29 +0800
Subject: [PATCH 750/881] process() takes timeUnitLength

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 051663c1..dd36c1d8 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -118,10 +118,14 @@ private static List<Integer> processTuningRoundTime(String appName,
 	}
 
 	private static Map<String, Integer> process(BufferedReader reader,
-			FileWriter writer, String startString) throws IOException {
+			FileWriter writer, String startString, boolean isms)
+			throws IOException {
 		String line;
 		String cfgPrefix = "1";
 		int i = 0;
+		int timeUnitLength = 1;
+		if (isms)
+			timeUnitLength = 2;
 		Map<String, Integer> ret = new HashMap<>(5000);
 		while ((line = reader.readLine()) != null) {
 			if (line.startsWith("----------------------------"))
@@ -129,7 +133,7 @@ private static Map<String, Integer> process(BufferedReader reader,
 			if (line.startsWith(startString)) {
 				String[] arr = line.split(" ");
 				String time = arr[3].trim();
-				time = time.substring(0, time.length() - 2);
+				time = time.substring(0, time.length() - timeUnitLength);
 				int val = Integer.parseInt(time);
 				ret.put(cfgPrefix, val);
 				String data = String

From d646955c535a768681f75b11b820e68a1142aab1 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:41:41 +0800
Subject: [PATCH 751/881] processTuningRoundTime() uses process()

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 23 ++++---------------
 1 file changed, 4 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index dd36c1d8..3eed2f65 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -33,7 +33,7 @@ private static Map<String, Integer> processCompileTime(String appName,
 
 		File outFile = new File(outDir, "processedCompileTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Total");
+		Map<String, Integer> ret = process(reader, writer, "Total", true);
 		reader.close();
 		writer.close();
 		return ret;
@@ -83,35 +83,20 @@ private static Map<String, Integer> processDrainTime(String appName,
 				String.format("%s%sdrainTime.txt", appName, File.separator)));
 		File outFile = new File(outDir, "processedDrainTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Drain time");
+		Map<String, Integer> ret = process(reader, writer, "Drain time", true);
 		writer.flush();
 		reader.close();
 		writer.close();
 		return ret;
 	}
 
-	private static List<Integer> processTuningRoundTime(String appName,
+	private static Map<String, Integer> processTuningRoundTime(String appName,
 			File outDir) throws IOException {
-
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
 		File outFile = new File(outDir, "processedTuningRoundTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		String line;
-		int i = 0;
-		List<Integer> ret = new ArrayList<Integer>(3000);
-		while ((line = reader.readLine()) != null) {
-			if (line.startsWith("Tuning")) {
-				String[] arr = line.split(" ");
-				String time = arr[4].trim();
-				time = time.substring(0, time.length() - 1);
-				int val = Integer.parseInt(time);
-				ret.add(val);
-				String data = String.format("%-6d\t%-6d\n", ++i, val);
-				writer.write(data);
-			}
-		}
-		writer.flush();
+		Map<String, Integer> ret = process(reader, writer, "Tuning", false);
 		reader.close();
 		writer.close();
 		return ret;

From d40dd84d58b0b4f809742b5eddfc98b2bc18c7ff Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:44:12 +0800
Subject: [PATCH 752/881] processRunTime() writes cfgPrefix

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 3eed2f65..fcd79719 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -53,10 +53,13 @@ private static List<Integer> processRunTime(String appName, File outDir)
 		File outFile = new File(outDir, "processedRunTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
+		String cfgPrefix = "1";
 		int i = 0;
 		List<Integer> ret = new ArrayList<Integer>(3000);
 		int min = Integer.MAX_VALUE;
 		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("----------------------------"))
+				cfgPrefix = cfgString(line);
 			if (line.startsWith("Execution")) {
 				String[] arr = line.split(" ");
 				String time = arr[3].trim();
@@ -66,8 +69,8 @@ private static List<Integer> processRunTime(String appName, File outDir)
 					val = 2 * min;
 				min = Math.min(min, val);
 				ret.add(val);
-				String data = String
-						.format("%-6d\t%-6d\t%-6d\n", ++i, val, min);
+				String data = String.format("%-6d\t%-6s\t%-6d\t%-6d\n", ++i,
+						cfgPrefix, val, min);
 				writer.write(data);
 			}
 		}

From 1871efa2fce061b2e087f24c6fcd0c8aaca06ea2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:47:02 +0800
Subject: [PATCH 753/881] processRunTime returns a map with cfgPrefix as key

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index fcd79719..a212f244 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -44,18 +44,16 @@ private static String cfgString(String line) {
 		return l.trim();
 	}
 
-	private static List<Integer> processRunTime(String appName, File outDir)
-			throws IOException {
-
+	private static Map<String, Integer> processRunTime(String appName,
+			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-
 		File outFile = new File(outDir, "processedRunTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		String cfgPrefix = "1";
 		int i = 0;
-		List<Integer> ret = new ArrayList<Integer>(3000);
+		Map<String, Integer> ret = new HashMap<>(5000);
 		int min = Integer.MAX_VALUE;
 		while ((line = reader.readLine()) != null) {
 			if (line.startsWith("----------------------------"))
@@ -68,7 +66,7 @@ private static List<Integer> processRunTime(String appName, File outDir)
 				if (val < 1)
 					val = 2 * min;
 				min = Math.min(min, val);
-				ret.add(val);
+				ret.put(cfgPrefix, val);
 				String data = String.format("%-6d\t%-6s\t%-6d\t%-6d\n", ++i,
 						cfgPrefix, val, min);
 				writer.write(data);

From 3c3d343dc17e9996e03f655f494f1414d5bba52d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 08:57:31 +0800
Subject: [PATCH 754/881] TimeLogProcessor summarize times based on cfgPrfix

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index a212f244..69ff9fa4 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -177,10 +177,11 @@ public static void summarize(String appName) throws IOException {
 		File summaryDir = new File(String.format("%s%ssummary", appName,
 				File.separator));
 		Utils.createDir(summaryDir.getPath());
-		List<Integer> compileTime = processCompileTime(appName, summaryDir);
-		List<Integer> runTime = processRunTime(appName, summaryDir);
-		List<Integer> drainTime = processDrainTime(appName, summaryDir);
-		List<Integer> tuningRoundTime = processTuningRoundTime(appName,
+		Map<String, Integer> compileTime = processCompileTime(appName,
+				summaryDir);
+		Map<String, Integer> runTime = processRunTime(appName, summaryDir);
+		Map<String, Integer> drainTime = processDrainTime(appName, summaryDir);
+		Map<String, Integer> tuningRoundTime = processTuningRoundTime(appName,
 				summaryDir);
 		String dataFile = "totalStats.txt";
 
@@ -191,12 +192,12 @@ public static void summarize(String appName) throws IOException {
 		FileWriter writer = new FileWriter(outfile, false);
 		int min = Integer.MAX_VALUE;
 
-		for (int i = 0; i < tuningRoundTime.size(); i++) {
-			min = Math.min(min, runTime.get(i));
+		for (int i = 1; i <= tuningRoundTime.size(); i++) {
+			String key = new Integer(i).toString();
+			min = Math.min(min, runTime.get(key));
 			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n",
-					i + 1, tuningRoundTime.get(i), compileTime.get(i),
-					runTime.get(i), drainTime.get(i), min);
-
+					i, tuningRoundTime.get(key), compileTime.get(key),
+					runTime.get(key), drainTime.get(key), min);
 			writer.write(msg);
 		}
 		writer.close();

From 85714cd8f5eebb8e95bfc3d79d9d99fd661ec932 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 09:14:26 +0800
Subject: [PATCH 755/881] process time based on the timepos in the string.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 69ff9fa4..43f35cd7 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -33,7 +33,7 @@ private static Map<String, Integer> processCompileTime(String appName,
 
 		File outFile = new File(outDir, "processedCompileTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Total", true);
+		Map<String, Integer> ret = process(reader, writer, "Total", true, 3);
 		reader.close();
 		writer.close();
 		return ret;
@@ -84,7 +84,8 @@ private static Map<String, Integer> processDrainTime(String appName,
 				String.format("%s%sdrainTime.txt", appName, File.separator)));
 		File outFile = new File(outDir, "processedDrainTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Drain time", true);
+		Map<String, Integer> ret = process(reader, writer, "Drain time", true,
+				3);
 		writer.flush();
 		reader.close();
 		writer.close();
@@ -97,14 +98,14 @@ private static Map<String, Integer> processTuningRoundTime(String appName,
 				String.format("%s%srunTime.txt", appName, File.separator)));
 		File outFile = new File(outDir, "processedTuningRoundTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Tuning", false);
+		Map<String, Integer> ret = process(reader, writer, "Tuning", false, 4);
 		reader.close();
 		writer.close();
 		return ret;
 	}
 
 	private static Map<String, Integer> process(BufferedReader reader,
-			FileWriter writer, String startString, boolean isms)
+			FileWriter writer, String startString, boolean isms, int timepos)
 			throws IOException {
 		String line;
 		String cfgPrefix = "1";
@@ -118,7 +119,7 @@ private static Map<String, Integer> process(BufferedReader reader,
 				cfgPrefix = cfgString(line);
 			if (line.startsWith(startString)) {
 				String[] arr = line.split(" ");
-				String time = arr[3].trim();
+				String time = arr[timepos].trim();
 				time = time.substring(0, time.length() - timeUnitLength);
 				int val = Integer.parseInt(time);
 				ret.put(cfgPrefix, val);
@@ -194,7 +195,12 @@ public static void summarize(String appName) throws IOException {
 
 		for (int i = 1; i <= tuningRoundTime.size(); i++) {
 			String key = new Integer(i).toString();
-			min = Math.min(min, runTime.get(key));
+			Integer time = runTime.get(key);
+			if (time == null) {
+				System.err.println("No running time for round " + key);
+			} else
+				min = Math.min(min, time);
+
 			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n",
 					i, tuningRoundTime.get(key), compileTime.get(key),
 					runTime.get(key), drainTime.get(key), min);
@@ -208,7 +214,6 @@ public static void summarize(String appName) throws IOException {
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
-
 	private static void makePlotFile(File dir, String name, String dataFile)
 			throws IOException {
 		File plotfile = new File(dir, "plot.plt");
@@ -238,6 +243,7 @@ private static void makePlotFile(File dir, String name, String dataFile)
 		writer.close();
 	}
 
+
 	private static void plot(File dir) throws IOException {
 		String[] s = { "/usr/bin/gnuplot", "plot.plt" };
 		try {

From f0127ad3d40b4b1df938c58da3c2bfac2a401c77 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 09:29:37 +0800
Subject: [PATCH 756/881] TimeLogProcessor writes verify.txt.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 43f35cd7..7f0311e6 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -186,26 +186,29 @@ public static void summarize(String appName) throws IOException {
 				summaryDir);
 		String dataFile = "totalStats.txt";
 
-		// String summaryDir = String.format("%s%ssummary", appName,
-		// File.separator);
-
 		File outfile = new File(summaryDir, dataFile);
 		FileWriter writer = new FileWriter(outfile, false);
+		FileWriter verify = new FileWriter(String.format("%s%sverify.txt",
+				appName, File.separator));
 		int min = Integer.MAX_VALUE;
 
 		for (int i = 1; i <= tuningRoundTime.size(); i++) {
 			String key = new Integer(i).toString();
 			Integer time = runTime.get(key);
+
 			if (time == null) {
 				System.err.println("No running time for round " + key);
-			} else
-				min = Math.min(min, time);
+			} else if (time < min) {
+				verify.write(String.format("%s=%d\n", key, time));
+				min = time;
+			}
 
 			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\t%-6d\t%-6d\n",
 					i, tuningRoundTime.get(key), compileTime.get(key),
 					runTime.get(key), drainTime.get(key), min);
 			writer.write(msg);
 		}
+		verify.close();
 		writer.close();
 
 		makePlotFile(summaryDir, appName, dataFile);
@@ -214,6 +217,7 @@ public static void summarize(String appName) throws IOException {
 		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
 		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
+
 	private static void makePlotFile(File dir, String name, String dataFile)
 			throws IOException {
 		File plotfile = new File(dir, "plot.plt");
@@ -243,7 +247,6 @@ private static void makePlotFile(File dir, String name, String dataFile)
 		writer.close();
 	}
 
-
 	private static void plot(File dir) throws IOException {
 		String[] s = { "/usr/bin/gnuplot", "plot.plt" };
 		try {

From 9e809c1eba8c0d61c3a4f98005f8e998f9a69559 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 09:31:25 +0800
Subject: [PATCH 757/881] New time's pos in the string is 3.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 7f0311e6..9449623f 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -98,7 +98,7 @@ private static Map<String, Integer> processTuningRoundTime(String appName,
 				String.format("%s%srunTime.txt", appName, File.separator)));
 		File outFile = new File(outDir, "processedTuningRoundTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
-		Map<String, Integer> ret = process(reader, writer, "Tuning", false, 4);
+		Map<String, Integer> ret = process(reader, writer, "Tuning", false, 3);
 		reader.close();
 		writer.close();
 		return ret;

From e42ddaf0f921da1846de6d83a0874c73b2914481 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 10:52:19 +0800
Subject: [PATCH 758/881] OnlineTuner refactored, inner class Verifier added

---
 .../distributed/runtimer/OnlineTuner.java     | 65 +++++++++++--------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 15865b9b..69ade8b6 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -59,7 +59,7 @@ public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
 		else if (GlobalConstants.tune == 2)
-			verifyTuningTimes(cfgPrefixes());
+			new Verifier().verify();
 		else
 			System.err
 					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
@@ -129,33 +129,6 @@ private void startTuner() throws IOException {
 		tuner.writeLine(Jsonifiers.toJson(app.getConfiguration()).toString());
 	}
 
-	/**
-	 * This method just picks a few configurations and re-run the app to ensure
-	 * the time we reported to the opentuner is correct.
-	 * 
-	 * This method can be called after the completion of the tuning.
-	 */
-	private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
-		for (String prefix : cfgPrefixes) {
-			String cfgName = String.format("%s_%s.cfg", prefix, app.name);
-			Configuration cfg = ConfigurationUtils.readConfiguration(app.name,
-					prefix);
-			if (cfg == null) {
-				System.err.println(String.format("No %s file exists", cfgName));
-				continue;
-			}
-			cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
-			evaluateConfig(cfg, cfgName);
-		}
-
-		try {
-			drainer.dumpDraindataStatistics();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-		terminate();
-	}
-
 	/**
 	 * TODO: Split this method into two methods, 1.reconfigure(),
 	 * 2.getFixedOutputTime().
@@ -370,4 +343,40 @@ private Iterable<String> cfgPrefixes() {
 		}
 		return cfgPrefixes;
 	}
+
+	private class Verifier {
+
+		public void verify() {
+			verifyTuningTimes(cfgPrefixes());
+		}
+
+		/**
+		 * This method just picks a few configurations and re-run the app to
+		 * ensure the time we reported to the opentuner is correct.
+		 * 
+		 * This method can be called after the completion of the tuning.
+		 */
+		private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
+			for (String prefix : cfgPrefixes) {
+				String cfgName = String.format("%s_%s.cfg", prefix, app.name);
+				Configuration cfg = ConfigurationUtils.readConfiguration(
+						app.name, prefix);
+				if (cfg == null) {
+					System.err.println(String.format("No %s file exists",
+							cfgName));
+					continue;
+				}
+				cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+				evaluateConfig(cfg, cfgName);
+			}
+
+			try {
+				drainer.dumpDraindataStatistics();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			terminate();
+		}
+
+	}
 }
\ No newline at end of file

From 1d2cf87447c9d4da267b4e93b6161624a7d7ac84 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 10:54:25 +0800
Subject: [PATCH 759/881] Method cfgPrefixes() has been moved to verifier

---
 .../distributed/runtimer/OnlineTuner.java     | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 69ade8b6..99799a09 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -323,27 +323,6 @@ private void evaluateConfig(Configuration cfg, String cfgName) {
 		}
 	}
 
-	private Iterable<String> cfgPrefixes() {
-		List<String> cfgPrefixes = new ArrayList<String>();
-		cfgPrefixes.add("final");
-		cfgPrefixes.add("hand");
-		try {
-			BufferedReader reader = new BufferedReader(new FileReader(
-					String.format("%s%sverify.txt", app.name, File.separator)));
-			String line;
-			while ((line = reader.readLine()) != null) {
-				String[] arr = line.split(",");
-				for (String s : arr) {
-					cfgPrefixes.add(s.trim());
-				}
-			}
-			reader.close();
-		} catch (IOException e) {
-			// e.printStackTrace();
-		}
-		return cfgPrefixes;
-	}
-
 	private class Verifier {
 
 		public void verify() {
@@ -378,5 +357,26 @@ private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
 			terminate();
 		}
 
+		private Iterable<String> cfgPrefixes() {
+			List<String> cfgPrefixes = new ArrayList<String>();
+			cfgPrefixes.add("final");
+			cfgPrefixes.add("hand");
+			try {
+				BufferedReader reader = new BufferedReader(new FileReader(
+						String.format("%s%sverify.txt", app.name,
+								File.separator)));
+				String line;
+				while ((line = reader.readLine()) != null) {
+					String[] arr = line.split(",");
+					for (String s : arr) {
+						cfgPrefixes.add(s.trim());
+					}
+				}
+				reader.close();
+			} catch (IOException e) {
+				// e.printStackTrace();
+			}
+			return cfgPrefixes;
+		}
 	}
 }
\ No newline at end of file

From d2188095a42c0135ee5288ed4f6e443646be4cb8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 10:56:54 +0800
Subject: [PATCH 760/881] evaluateConfig() has been moved to verifier.

---
 .../distributed/runtimer/OnlineTuner.java     | 92 ++++++++++---------
 1 file changed, 47 insertions(+), 45 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 99799a09..7563d6ac 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -278,51 +278,6 @@ private void terminate() {
 		}
 	}
 
-	/**
-	 * Evaluates a configuration.
-	 * 
-	 * @param cfg
-	 *            configuration that needs to be evaluated
-	 * @param cfgName
-	 *            name of the configuration. This is just for logging purpose.
-	 */
-	private void evaluateConfig(Configuration cfg, String cfgName) {
-		System.out.println("Evaluating " + cfgName);
-		FileWriter writer;
-		double total = 0;
-		int count = 2;
-		try {
-			writer = new FileWriter(String.format("%s%sEval_%s.txt", app.name,
-					File.separator, app.name), true);
-			writer.write("\n----------------------------------------\n");
-			writer.write(String.format("Configuration name = %s\n", cfgName));
-			if (cfg != null) {
-				Pair<Boolean, Long> ret;
-				for (int i = 0; i < count; i++) {
-					logger.newConfiguration(cfgName);
-					ret = reconfigure(cfg, 0);
-					if (ret.first) {
-						prognosticator.time(ret.second);
-						writer.write(ret.second.toString());
-						writer.write('\n');
-						writer.flush();
-						total += ret.second;
-					} else {
-						break;
-					}
-				}
-				double avg = total / count;
-				writer.write(String.format("Average execution time = %f%n\n",
-						avg));
-			} else {
-				writer.write("Null configuration\n");
-			}
-			writer.close();
-		} catch (IOException e1) {
-			e1.printStackTrace();
-		}
-	}
-
 	private class Verifier {
 
 		public void verify() {
@@ -378,5 +333,52 @@ private Iterable<String> cfgPrefixes() {
 			}
 			return cfgPrefixes;
 		}
+
+		/**
+		 * Evaluates a configuration.
+		 * 
+		 * @param cfg
+		 *            configuration that needs to be evaluated
+		 * @param cfgName
+		 *            name of the configuration. This is just for logging
+		 *            purpose.
+		 */
+		private void evaluateConfig(Configuration cfg, String cfgName) {
+			System.out.println("Evaluating " + cfgName);
+			FileWriter writer;
+			double total = 0;
+			int count = 2;
+			try {
+				writer = new FileWriter(String.format("%s%sEval_%s.txt",
+						app.name, File.separator, app.name), true);
+				writer.write("\n----------------------------------------\n");
+				writer.write(String
+						.format("Configuration name = %s\n", cfgName));
+				if (cfg != null) {
+					Pair<Boolean, Long> ret;
+					for (int i = 0; i < count; i++) {
+						logger.newConfiguration(cfgName);
+						ret = reconfigure(cfg, 0);
+						if (ret.first) {
+							prognosticator.time(ret.second);
+							writer.write(ret.second.toString());
+							writer.write('\n');
+							writer.flush();
+							total += ret.second;
+						} else {
+							break;
+						}
+					}
+					double avg = total / count;
+					writer.write(String.format(
+							"Average execution time = %f%n\n", avg));
+				} else {
+					writer.write("Null configuration\n");
+				}
+				writer.close();
+			} catch (IOException e1) {
+				e1.printStackTrace();
+			}
+		}
 	}
 }
\ No newline at end of file

From d99fd2595e374f002fb7b3e11a71c45a796c92b4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 11:02:29 +0800
Subject: [PATCH 761/881] verifyTuningTimes() takes a map as arg

Map of cfgPrefix and expected running time
---
 .../distributed/runtimer/OnlineTuner.java     | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 7563d6ac..739e35f1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -5,8 +5,8 @@
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -289,9 +289,12 @@ public void verify() {
 		 * ensure the time we reported to the opentuner is correct.
 		 * 
 		 * This method can be called after the completion of the tuning.
+		 * 
+		 * @param cfgPrefixes
+		 *            map of cfgPrefixes and expected running time.
 		 */
-		private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
-			for (String prefix : cfgPrefixes) {
+		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
+			for (String prefix : cfgPrefixes.keySet()) {
 				String cfgName = String.format("%s_%s.cfg", prefix, app.name);
 				Configuration cfg = ConfigurationUtils.readConfiguration(
 						app.name, prefix);
@@ -312,10 +315,10 @@ private void verifyTuningTimes(Iterable<String> cfgPrefixes) {
 			terminate();
 		}
 
-		private Iterable<String> cfgPrefixes() {
-			List<String> cfgPrefixes = new ArrayList<String>();
-			cfgPrefixes.add("final");
-			cfgPrefixes.add("hand");
+		private Map<String, Integer> cfgPrefixes() {
+			Map<String, Integer> cfgPrefixes = new HashMap<>();
+			cfgPrefixes.put("final", 0);
+			cfgPrefixes.put("hand", 0);
 			try {
 				BufferedReader reader = new BufferedReader(new FileReader(
 						String.format("%s%sverify.txt", app.name,
@@ -324,12 +327,11 @@ private Iterable<String> cfgPrefixes() {
 				while ((line = reader.readLine()) != null) {
 					String[] arr = line.split(",");
 					for (String s : arr) {
-						cfgPrefixes.add(s.trim());
+						cfgPrefixes.put(s.trim(), 0);
 					}
 				}
 				reader.close();
 			} catch (IOException e) {
-				// e.printStackTrace();
 			}
 			return cfgPrefixes;
 		}

From 98a2704ee0127b71dbf4ea88c977b013335d2660 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 14:14:20 +0800
Subject: [PATCH 762/881] cfgPrefixes() handles timeLogProcessor's output.

cfgPrefixes() handles both manually entered and TimeLogProcessor generated
configurations.
---
 .../distributed/runtimer/OnlineTuner.java     | 34 ++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 739e35f1..277f1621 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -325,10 +325,10 @@ private Map<String, Integer> cfgPrefixes() {
 								File.separator)));
 				String line;
 				while ((line = reader.readLine()) != null) {
-					String[] arr = line.split(",");
-					for (String s : arr) {
-						cfgPrefixes.put(s.trim(), 0);
-					}
+					if (line.contains("="))
+						process1(line, cfgPrefixes);
+					else
+						process2(line, cfgPrefixes);
 				}
 				reader.close();
 			} catch (IOException e) {
@@ -336,6 +336,32 @@ private Map<String, Integer> cfgPrefixes() {
 			return cfgPrefixes;
 		}
 
+		/**
+		 * Processes the line that is generated by {@link TimeLogProcessor}.
+		 */
+		private void process1(String line, Map<String, Integer> cfgPrefixes) {
+			String[] arr = line.split("=");
+			String cfgPrefix = arr[0].trim();
+			int expectedTime = 0;
+			if (arr.length > 1)
+				try {
+					expectedTime = Integer.parseInt(arr[1]);
+				} catch (NumberFormatException ex) {
+					System.err.println("NumberFormatException: " + arr[1]);
+				}
+			cfgPrefixes.put(cfgPrefix, expectedTime);
+		}
+
+		/**
+		 * Processes manually entered lines in the verify.txt
+		 */
+		private void process2(String line, Map<String, Integer> cfgPrefixes) {
+			String[] arr = line.split(",");
+			for (String s : arr) {
+				cfgPrefixes.put(s.trim(), 0);
+			}
+		}
+
 		/**
 		 * Evaluates a configuration.
 		 * 

From 8d5f9cf12cd50a2f545a5d36b4a8c63cff88174e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 14:20:05 +0800
Subject: [PATCH 763/881] Terminating the app is removed from verifier

Verifier will not terminate the app after the verification. The caller must
termintate the app.
---
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java   | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 277f1621..f4edc778 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -306,13 +306,6 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 				cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
 				evaluateConfig(cfg, cfgName);
 			}
-
-			try {
-				drainer.dumpDraindataStatistics();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-			terminate();
 		}
 
 		private Map<String, Integer> cfgPrefixes() {

From 1e71cbef03f3fac5a31cfaaeb43874643c7bcb69 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 14:43:31 +0800
Subject: [PATCH 764/881] verifyTuningTimes() updates the time to Eval.txt

---
 .../distributed/runtimer/OnlineTuner.java     | 36 +++++++++++--------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index f4edc778..5b537e5d 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -294,17 +294,27 @@ public void verify() {
 		 *            map of cfgPrefixes and expected running time.
 		 */
 		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
-			for (String prefix : cfgPrefixes.keySet()) {
-				String cfgName = String.format("%s_%s.cfg", prefix, app.name);
-				Configuration cfg = ConfigurationUtils.readConfiguration(
-						app.name, prefix);
-				if (cfg == null) {
-					System.err.println(String.format("No %s file exists",
-							cfgName));
-					continue;
+			try {
+				FileWriter writer = new FileWriter(String.format(
+						"%s%sEval_%s.txt", app.name, File.separator, app.name),
+						true);
+				for (String prefix : cfgPrefixes.keySet()) {
+					String cfgName = String.format("%s_%s.cfg", prefix,
+							app.name);
+					Configuration cfg = ConfigurationUtils.readConfiguration(
+							app.name, prefix);
+					if (cfg == null) {
+						System.err.println(String.format("No %s file exists",
+								cfgName));
+						continue;
+					}
+					cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+					evaluateConfig(cfg, cfgName, writer);
 				}
-				cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
-				evaluateConfig(cfg, cfgName);
+
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
 			}
 		}
 
@@ -364,14 +374,12 @@ private void process2(String line, Map<String, Integer> cfgPrefixes) {
 		 *            name of the configuration. This is just for logging
 		 *            purpose.
 		 */
-		private void evaluateConfig(Configuration cfg, String cfgName) {
+		private void evaluateConfig(Configuration cfg, String cfgName,
+				FileWriter writer) {
 			System.out.println("Evaluating " + cfgName);
-			FileWriter writer;
 			double total = 0;
 			int count = 2;
 			try {
-				writer = new FileWriter(String.format("%s%sEval_%s.txt",
-						app.name, File.separator, app.name), true);
 				writer.write("\n----------------------------------------\n");
 				writer.write(String
 						.format("Configuration name = %s\n", cfgName));

From d5a622ff5a042fccc7f8052a1f25584eb7843e28 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 14:46:00 +0800
Subject: [PATCH 765/881] Writes header info to the Eval_.txt.

---
 .../impl/distributed/runtimer/OnlineTuner.java      | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 5b537e5d..98272108 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -7,6 +7,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -295,9 +296,7 @@ public void verify() {
 		 */
 		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 			try {
-				FileWriter writer = new FileWriter(String.format(
-						"%s%sEval_%s.txt", app.name, File.separator, app.name),
-						true);
+				FileWriter writer = writer();
 				for (String prefix : cfgPrefixes.keySet()) {
 					String cfgName = String.format("%s_%s.cfg", prefix,
 							app.name);
@@ -317,6 +316,14 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 				e.printStackTrace();
 			}
 		}
+		private FileWriter writer() throws IOException {
+			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
+					app.name, File.separator, app.name), true);
+			writer.write("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+			Properties prop = GlobalConstants.getProperties();
+			prop.store(writer, "");
+			return writer;
+		}
 
 		private Map<String, Integer> cfgPrefixes() {
 			Map<String, Integer> cfgPrefixes = new HashMap<>();

From 4e29c1af7f767772d06edfe2154b514185d66f02 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 14:53:13 +0800
Subject: [PATCH 766/881] verifyTuningTimes writes cfgName & expected Time

---
 .../impl/distributed/runtimer/OnlineTuner.java     | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 98272108..c8f3838c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -297,7 +297,9 @@ public void verify() {
 		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 			try {
 				FileWriter writer = writer();
-				for (String prefix : cfgPrefixes.keySet()) {
+				for (Map.Entry<String, Integer> en : cfgPrefixes.entrySet()) {
+					String prefix = en.getKey();
+					Integer expectedRunningTime = en.getValue();
 					String cfgName = String.format("%s_%s.cfg", prefix,
 							app.name);
 					Configuration cfg = ConfigurationUtils.readConfiguration(
@@ -308,14 +310,19 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 						continue;
 					}
 					cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+					writer.write("----------------------------------------\n");
+					writer.write(String.format("Configuration name = %s\n",
+							cfgName));
+					writer.write(String.format("Expected running time = %d\n",
+							expectedRunningTime));
 					evaluateConfig(cfg, cfgName, writer);
 				}
 
 			} catch (IOException e) {
-				// TODO Auto-generated catch block
 				e.printStackTrace();
 			}
 		}
+
 		private FileWriter writer() throws IOException {
 			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
 					app.name, File.separator, app.name), true);
@@ -387,9 +394,6 @@ private void evaluateConfig(Configuration cfg, String cfgName,
 			double total = 0;
 			int count = 2;
 			try {
-				writer.write("\n----------------------------------------\n");
-				writer.write(String
-						.format("Configuration name = %s\n", cfgName));
 				if (cfg != null) {
 					Pair<Boolean, Long> ret;
 					for (int i = 0; i < count; i++) {

From 8b0fda3e39452e8d21826c3aa2b6ca7c7d8ec9fa Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 15:05:16 +0800
Subject: [PATCH 767/881] evaluateConfig() returns running time list.

---
 .../distributed/runtimer/OnlineTuner.java     | 45 ++++++++-----------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index c8f3838c..d7d4eab0 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -5,7 +5,9 @@
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -315,7 +317,7 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 							cfgName));
 					writer.write(String.format("Expected running time = %d\n",
 							expectedRunningTime));
-					evaluateConfig(cfg, cfgName, writer);
+					List<Long> runningTime = evaluateConfig(cfg, cfgName);
 				}
 
 			} catch (IOException e) {
@@ -388,37 +390,26 @@ private void process2(String line, Map<String, Integer> cfgPrefixes) {
 		 *            name of the configuration. This is just for logging
 		 *            purpose.
 		 */
-		private void evaluateConfig(Configuration cfg, String cfgName,
-				FileWriter writer) {
+		private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			System.out.println("Evaluating " + cfgName);
-			double total = 0;
 			int count = 2;
-			try {
-				if (cfg != null) {
-					Pair<Boolean, Long> ret;
-					for (int i = 0; i < count; i++) {
-						logger.newConfiguration(cfgName);
-						ret = reconfigure(cfg, 0);
-						if (ret.first) {
-							prognosticator.time(ret.second);
-							writer.write(ret.second.toString());
-							writer.write('\n');
-							writer.flush();
-							total += ret.second;
-						} else {
-							break;
-						}
+			List<Long> runningTime = new ArrayList<>(count);
+			Pair<Boolean, Long> ret;
+			if (cfg != null) {
+				for (int i = 0; i < count; i++) {
+					logger.newConfiguration(cfgName);
+					ret = reconfigure(cfg, 0);
+					if (ret.first) {
+						prognosticator.time(ret.second);
+						runningTime.add(ret.second);
+					} else {
+						System.err.println("Evaluation failed...");
 					}
-					double avg = total / count;
-					writer.write(String.format(
-							"Average execution time = %f%n\n", avg));
-				} else {
-					writer.write("Null configuration\n");
 				}
-				writer.close();
-			} catch (IOException e1) {
-				e1.printStackTrace();
+			} else {
+				System.err.println("Null configuration\n");
 			}
+			return runningTime;
 		}
 	}
 }
\ No newline at end of file

From f503ab2aaaa4ee0c92e4840a5ecccd97bbf30d22 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 15:17:21 +0800
Subject: [PATCH 768/881] processRunningTimes() added.

---
 .../distributed/runtimer/OnlineTuner.java     | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index d7d4eab0..11347225 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -315,16 +315,36 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 					writer.write("----------------------------------------\n");
 					writer.write(String.format("Configuration name = %s\n",
 							cfgName));
-					writer.write(String.format("Expected running time = %d\n",
-							expectedRunningTime));
-					List<Long> runningTime = evaluateConfig(cfg, cfgName);
+					List<Long> runningTimes = evaluateConfig(cfg, cfgName);
+					processRunningTimes(runningTimes, expectedRunningTime,
+							writer);
 				}
-
+				writer.close();
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
 		}
 
+		private void processRunningTimes(List<Long> runningTimes,
+				Integer expectedRunningTime, FileWriter writer)
+				throws IOException {
+			writer.write(String.format("Expected running time = %dms\n",
+					expectedRunningTime));
+			int correctEval = 0;
+			double total = 0;
+			for (int i = 0; i < runningTimes.size(); i++) {
+				long runningTime = runningTimes.get(i);
+				if (runningTime > 0) {
+					correctEval++;
+					total += runningTime;
+				}
+				writer.write(String.format("Evaluation %d = %dms\n", i + 1,
+						runningTime));
+			}
+			double avg = total / correctEval;
+			writer.write(String.format("Average running time = %.3fms\n", avg));
+		}
+
 		private FileWriter writer() throws IOException {
 			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
 					app.name, File.separator, app.name), true);

From 4624be60db2b0a2c0658ede8c04b2802840673b6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 15:19:37 +0800
Subject: [PATCH 769/881] terminates the app after the verification.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 11347225..62d6e989 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -61,9 +61,10 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 	public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
-		else if (GlobalConstants.tune == 2)
+		else if (GlobalConstants.tune == 2) {
 			new Verifier().verify();
-		else
+			terminate();
+		} else
 			System.err
 					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
 	}

From 290ccdb139f6d44d238f861a60c0c5ee17f932a0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 16:06:16 +0800
Subject: [PATCH 770/881] handleTermination calls verifier.verify().

---
 .../impl/distributed/runtimer/OnlineTuner.java         | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 62d6e989..071cd08c 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -43,6 +43,7 @@ public class OnlineTuner implements Runnable {
 	private final boolean needTermination;
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
+	private final Verifier verifier;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
@@ -55,6 +56,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.needTermination = needTermination;
 		this.logger = logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
+		this.verifier = new Verifier();
 	}
 
 	@Override
@@ -62,7 +64,7 @@ public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
 		else if (GlobalConstants.tune == 2) {
-			new Verifier().verify();
+			verifier.verify();
 			terminate();
 		} else
 			System.err
@@ -224,12 +226,8 @@ private void handleTermination() throws IOException {
 		ConfigurationUtils.saveConfg(finalConfg, "final", app.name);
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
-		evaluateConfig(finalcfg, "Final configuration");
 
-		Configuration handCfg = ConfigurationUtils.readConfiguration(app.name,
-				"hand");
-		handCfg = ConfigurationUtils.addConfigPrefix(handCfg, "hand");
-		evaluateConfig(handCfg, "Handtuned configuration");
+		verifier.verify();
 
 		if (needTermination) {
 			terminate();

From c9f943778a89b3d0806294b6016332e1826b250d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 16:24:53 +0800
Subject: [PATCH 771/881] Eval.txt's header line changed.

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 071cd08c..08852521 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -318,6 +318,7 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 					processRunningTimes(runningTimes, expectedRunningTime,
 							writer);
 				}
+				writer.write("**************FINISHED**************\n\n");
 				writer.close();
 			} catch (IOException e) {
 				e.printStackTrace();
@@ -347,7 +348,7 @@ private void processRunningTimes(List<Long> runningTimes,
 		private FileWriter writer() throws IOException {
 			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
 					app.name, File.separator, app.name), true);
-			writer.write("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+			writer.write("##########################################################");
 			Properties prop = GlobalConstants.getProperties();
 			prop.store(writer, "");
 			return writer;

From ae69adfe52943fb38f4e97f2fa22480b57fbec37 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 16:55:16 +0800
Subject: [PATCH 772/881] Utils backups ReadMe.txt before creating new.

---
 .../mit/streamjit/impl/distributed/common/Utils.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index bf06f53e..b36d99fa 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -133,6 +133,7 @@ public static boolean createAppDir(String appName) {
 	 */
 	public static void writeReadMeTxt(String appName) {
 		try {
+			backupReadMeTxt(appName);
 			FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
 					appName, File.separator));
 			DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
@@ -147,6 +148,17 @@ public static void writeReadMeTxt(String appName) {
 		}
 	}
 
+	private static void backupReadMeTxt(String appName) {
+		File readMeOrig = new File(String.format("%s%sREADMEOrig.txt", appName,
+				File.separator));
+		File readMe = new File(String.format("%s%sREADME.txt", appName,
+				File.separator));
+		if (readMeOrig.exists())
+			return;
+		if (readMe.exists())
+			readMe.renameTo(readMeOrig);
+	}
+
 	/**
 	 * Creates and returns a {@link FileWriter} with append = false. Suppresses
 	 * {@link IOException} and returns null if exception occurred. This method

From 17b391d8495e62365c3305d5e89139cdb3f5282d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 17:03:05 +0800
Subject: [PATCH 773/881] Utils.rename() added.

---
 .../impl/distributed/common/Utils.java         | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index b36d99fa..c505199c 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -133,7 +133,7 @@ public static boolean createAppDir(String appName) {
 	 */
 	public static void writeReadMeTxt(String appName) {
 		try {
-			backupReadMeTxt(appName);
+			rename(appName, "README.txt");
 			FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
 					appName, File.separator));
 			DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
@@ -148,15 +148,15 @@ public static void writeReadMeTxt(String appName) {
 		}
 	}
 
-	private static void backupReadMeTxt(String appName) {
-		File readMeOrig = new File(String.format("%s%sREADMEOrig.txt", appName,
-				File.separator));
-		File readMe = new File(String.format("%s%sREADME.txt", appName,
-				File.separator));
-		if (readMeOrig.exists())
+	private static void rename(String appName, String fileName) {
+		File file = new File(String.format("%s%s%s", appName, File.separator,
+				fileName));
+		File fileOrig = new File(String.format("%s%s%s.Orig", appName,
+				File.separator, fileName));
+		if (fileOrig.exists())
 			return;
-		if (readMe.exists())
-			readMe.renameTo(readMeOrig);
+		if (file.exists())
+			file.renameTo(fileOrig);
 	}
 
 	/**

From d716f3f7e54520945a4493d6a93691f5abbe14c8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 17:04:56 +0800
Subject: [PATCH 774/881] Utils.rename() returns true if renaming succeeded.

---
 src/edu/mit/streamjit/impl/distributed/common/Utils.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index c505199c..0b2f5ad3 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -148,15 +148,19 @@ public static void writeReadMeTxt(String appName) {
 		}
 	}
 
-	private static void rename(String appName, String fileName) {
+	/**
+	 * @return true iff renaming is success.
+	 */
+	private static boolean rename(String appName, String fileName) {
 		File file = new File(String.format("%s%s%s", appName, File.separator,
 				fileName));
 		File fileOrig = new File(String.format("%s%s%s.Orig", appName,
 				File.separator, fileName));
 		if (fileOrig.exists())
-			return;
+			return false;
 		if (file.exists())
 			file.renameTo(fileOrig);
+		return true;
 	}
 
 	/**

From 3ce187b9a56db43b9dbc5f1d86f0ce1ae9187b02 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 17:12:42 +0800
Subject: [PATCH 775/881] Verifier backups the files before start evaluating

---
 .../impl/distributed/common/Utils.java        |  2 +-
 .../distributed/runtimer/OnlineTuner.java     | 21 ++++++++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 0b2f5ad3..3cb6e664 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -151,7 +151,7 @@ public static void writeReadMeTxt(String appName) {
 	/**
 	 * @return true iff renaming is success.
 	 */
-	private static boolean rename(String appName, String fileName) {
+	public static boolean rename(String appName, String fileName) {
 		File file = new File(String.format("%s%s%s", appName, File.separator,
 				fileName));
 		File fileOrig = new File(String.format("%s%s%s.Orig", appName,
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 08852521..b15db2c8 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -20,6 +20,7 @@
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
@@ -64,7 +65,7 @@ public void run() {
 		if (GlobalConstants.tune == 1)
 			tune();
 		else if (GlobalConstants.tune == 2) {
-			verifier.verify();
+			verifier.verify(true);
 			terminate();
 		} else
 			System.err
@@ -227,7 +228,7 @@ private void handleTermination() throws IOException {
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
 
-		verifier.verify();
+		verifier.verify(false);
 
 		if (needTermination) {
 			terminate();
@@ -282,7 +283,9 @@ private void terminate() {
 
 	private class Verifier {
 
-		public void verify() {
+		public void verify(boolean needBackup) {
+			if (needBackup)
+				backup();
 			verifyTuningTimes(cfgPrefixes());
 		}
 
@@ -431,5 +434,17 @@ private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			}
 			return runningTime;
 		}
+
+		/**
+		 * Backups the files generated during tuning.
+		 */
+		private void backup() {
+			Utils.rename(app.name, "summary");
+			Utils.rename(app.name, "compileTime.txt");
+			Utils.rename(app.name, "runTime.txt");
+			Utils.rename(app.name, "drainTime.txt");
+			Utils.rename(app.name, "GraphProperty.txt");
+			Utils.rename(app.name, "profile.txt");
+		}
 	}
 }
\ No newline at end of file

From e6c6dae6030f601563a3f9f84493863c6ca3dc32 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 18:53:05 +0800
Subject: [PATCH 776/881] Orig->orig.

---
 src/edu/mit/streamjit/impl/distributed/common/Utils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 3cb6e664..295f6389 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -154,7 +154,7 @@ public static void writeReadMeTxt(String appName) {
 	public static boolean rename(String appName, String fileName) {
 		File file = new File(String.format("%s%s%s", appName, File.separator,
 				fileName));
-		File fileOrig = new File(String.format("%s%s%s.Orig", appName,
+		File fileOrig = new File(String.format("%s%s%s.orig", appName,
 				File.separator, fileName));
 		if (fileOrig.exists())
 			return false;

From 6ee2aeec56cf012ec3ef2e6cc838f891c482ddae Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 20:32:36 +0800
Subject: [PATCH 777/881] Ant target createDownloadScript added.

---
 jarapp.xml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index 14bb2c26..fa17bef2 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -100,7 +100,7 @@
 		<!--echo message="Password: ${password}" /-->
 	</target>
 
-	<target name="remotesetup" depends="sendtoLanka">
+	<target name="remotesetup" depends="sendtoLanka, createDownloadScript">
 		<sshexec trust="true" host="lanka.csail.mit.edu" username="${username}" password="${password}" command="/data/scratch/${username}/setup.sh ${app} ${streamGraphName} ${numOfSNs}" />
 	</target>
 
@@ -131,4 +131,21 @@
 	<target name="compile" depends="clean, makedir">
 		<javac srcdir="${src.dir}" destdir="${build.dir}" classpathref="build.classpath" />
 	</target>
+
+	<target name="createDownloadScript">
+		<echo file="../Experiments/${app}.sh" append="false">#!/bin/bash
+#Author - Sumanan
+app=${app}
+mainClass=${streamGraphName}
+if [ -d $app ]; then
+	echo "$app exists. No downloads..."
+ 	exit
+fi
+mkdir -p $app
+cd $app
+mkdir -p $mainClass
+scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
+scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/\{*.sh,slurm-*,options.properties,*.jar\} .
+	</echo>
+	</target>
 </project>

From 64fc2b3a75aaf6d26ff16c21b76a04f89e319f5f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 20:35:30 +0800
Subject: [PATCH 778/881] A sample file to download files from Lanka cluster

---
 scripts/FMRadio.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 scripts/FMRadio.sh

diff --git a/scripts/FMRadio.sh b/scripts/FMRadio.sh
new file mode 100644
index 00000000..988d2c77
--- /dev/null
+++ b/scripts/FMRadio.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#Author - Sumanan
+# A sample file to download tuning files from Lanka cluster.
+app=FMRadio15
+mainClass=FMRadioCore
+if [ -d $app ]; then
+	echo "$app exists. No downloads..."
+ 	exit
+fi
+mkdir -p $app
+cd $app
+mkdir -p $mainClass
+scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
+scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/\{*.sh,slurm-*,options.properties,*.jar\} .
+	

From 2a338d385d274d37143cd3a0807dc24dd749b79f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 28 Feb 2015 22:57:15 +0800
Subject: [PATCH 779/881] Ant: sumanan -> ${username}.

---
 jarapp.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jarapp.xml b/jarapp.xml
index fa17bef2..c57aca79 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -144,8 +144,8 @@ fi
 mkdir -p $app
 cd $app
 mkdir -p $mainClass
-scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
-scp -r sumanan@lanka.csail.mit.edu:/data/scratch/sumanan/$app/\{*.sh,slurm-*,options.properties,*.jar\} .
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties,*.jar\} .
 	</echo>
 	</target>
 </project>

From 7250bd2170c9e0d3dcfc2f77816a4e09876aa0e2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Mar 2015 21:02:36 +0800
Subject: [PATCH 780/881] Stopped tuning maxNumCores.

---
 .../mit/streamjit/impl/distributed/DistributedBlobFactory.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index a74ea020..0d60da04 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -78,7 +78,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			builder.addParameter(p);
 
 		connectionManager.addConnectionParameters(builder, workers);
-		addMaxCoreParam(builder);
+		// addMaxCoreParam(builder);
 		return builder.build();
 	}
 

From 64bfc90ca43493abfd84d3e57ffbdd5f5ee8d482 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Mar 2015 22:34:26 +0800
Subject: [PATCH 781/881] Moved program options to Options.java

Moved program options from GlobalConstants.java to Options.java. Git detects
the new file Options.java as a renamed file of GlobalConstants.java. So I will
add Options.java in a new commit.
---
 .../impl/common/drainer/AbstractDrainer.java  |  10 +-
 .../impl/common/drainer/BlobGraph.java        |   6 +-
 .../streamjit/impl/compiler2/Compiler2.java   |   4 +-
 .../distributed/DistributedBlobFactory.java   |   8 +-
 .../DistributedStreamCompiler.java            |   8 +-
 .../impl/distributed/StreamJitApp.java        |   9 +-
 .../impl/distributed/StreamJitAppManager.java |  11 +-
 .../impl/distributed/TailChannels.java        |  22 +--
 .../distributed/common/GlobalConstants.java   | 179 ------------------
 .../impl/distributed/common/Utils.java        |   2 +-
 .../distributed/runtimer/OnlineTuner.java     |  10 +-
 src/edu/mit/streamjit/tuner/TCPTuner.java     |   6 +-
 12 files changed, 49 insertions(+), 226 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index dbe72bbc..e2a73024 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -32,7 +32,7 @@
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
@@ -192,7 +192,7 @@ public final boolean startDraining(int type) {
 				throw e;
 			}
 
-			if (GlobalConstants.needDrainDeadlockHandler)
+			if (Options.needDrainDeadlockHandler)
 				this.schExecutorService = Executors
 						.newSingleThreadScheduledExecutor();
 
@@ -287,7 +287,7 @@ public final void drained(Token blobID) {
 	 * @throws InterruptedException
 	 */
 	public final void awaitDrainData() throws InterruptedException {
-		if (GlobalConstants.useDrainData) {
+		if (Options.useDrainData) {
 			drainDataLatch.await();
 			app.drainData = getDrainData();
 		}
@@ -308,7 +308,7 @@ public final void newSNDrainData(SNDrainedData snDrainedData) {
 	 * @return Aggregated DrainData after the draining.
 	 */
 	private final DrainData getDrainData() {
-		if (!GlobalConstants.useDrainData)
+		if (!Options.useDrainData)
 			return null;
 		DrainData drainData = null;
 		Map<Token, ImmutableList<Object>> boundaryInputData = new HashMap<>();
@@ -550,7 +550,7 @@ void drainingDone(BlobNode blobNode) {
 				state = DrainerState.NODRAINING;
 			}
 
-			if (GlobalConstants.needDrainDeadlockHandler)
+			if (Options.needDrainDeadlockHandler)
 				schExecutorService.shutdownNow();
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
index 73e3117c..7bbb38b7 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/BlobGraph.java
@@ -25,7 +25,7 @@
 import edu.mit.streamjit.impl.common.IOInfo;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer.DrainerState;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 
 /**
@@ -321,7 +321,7 @@ void drain() {
 			}
 
 			DrainType drainType;
-			if (GlobalConstants.useDrainData)
+			if (Options.useDrainData)
 				if (drainer.state == DrainerState.FINAL)
 					drainType = DrainType.FINAL;
 				else
@@ -332,7 +332,7 @@ void drain() {
 			drainer.drain(blobID, drainType);
 
 			// TODO: Verify the waiting time is reasonable.
-			if (GlobalConstants.needDrainDeadlockHandler)
+			if (Options.needDrainDeadlockHandler)
 				drainer.schExecutorService.schedule(deadLockHandler(), 6000,
 						TimeUnit.MILLISECONDS);
 		}
diff --git a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
index 05cf2f2a..ccafd1f5 100644
--- a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
+++ b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
@@ -50,7 +50,7 @@
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.DrainInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.ReadInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.WriteInstruction;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
@@ -99,7 +99,7 @@ public class Compiler2 {
 	public static final RemovalStrategy REMOVAL_STRATEGY = new BitsetRemovalStrategy();
 	public static final FusionStrategy FUSION_STRATEGY = new BitsetFusionStrategy();
 	public static final UnboxingStrategy UNBOXING_STRATEGY = new BitsetUnboxingStrategy();
-	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(GlobalConstants.maxNumCores);
+	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(Options.maxNumCores);
 	public static final StorageStrategy INTERNAL_STORAGE_STRATEGY = new TuneInternalStorageStrategy();
 	public static final StorageStrategy EXTERNAL_STORAGE_STRATEGY = new TuneExternalStorageStrategy();
 	private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup();
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 0d60da04..134c727c 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -11,7 +11,7 @@
 import edu.mit.streamjit.impl.compiler.CompilerBlobFactory;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.interp.Interpreter.InterpreterBlobFactory;
 
 /**
@@ -68,7 +68,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 		Configuration distCfg = partitionManager.getDefaultConfiguration(
 				workers, noOfMachines);
-		if (!GlobalConstants.useCompilerBlob)
+		if (!Options.useCompilerBlob)
 			return distCfg;
 
 		Configuration.Builder builder = Configuration.builder(distCfg);
@@ -84,10 +84,10 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 
 	private void addMaxCoreParam(Configuration.Builder builder) {
 		int min = 1;
-		int val = GlobalConstants.maxNumCores / 2;
+		int val = Options.maxNumCores / 2;
 		val = min > val ? min : val;
 		Parameter p = new Configuration.IntParameter("maxNumCores", min,
-				GlobalConstants.maxNumCores, val);
+				Options.maxNumCores, val);
 		builder.addParameter(p);
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 36a13fe7..0d1e5d17 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -30,7 +30,7 @@
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.concurrent.ConcurrentStreamCompiler;
 import edu.mit.streamjit.impl.distributed.HeadChannel.HeadBuffer;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -100,7 +100,7 @@ public DistributedStreamCompiler(int noOfnodes) {
 	public DistributedStreamCompiler(int noOfnodes, Configuration cfg) {
 		if (noOfnodes < 1)
 			throw new IllegalArgumentException("noOfnodes must be 1 or greater");
-		if (GlobalConstants.singleNodeOnline) {
+		if (Options.singleNodeOnline) {
 			System.out
 					.println("Flag GlobalConstants.singleNodeOnline is enabled."
 							+ " noOfNodes passed as compiler argument has no effect");
@@ -136,7 +136,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		manager.reconfigure(1);
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
-		if (GlobalConstants.tune > 0 && this.cfg != null) {
+		if (Options.tune > 0 && this.cfg != null) {
 			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
 					cfgManager, logger, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
@@ -269,7 +269,7 @@ private <I, O> void setConfiguration(Controller controller,
 				controller.closeAll();
 				throw new IllegalConfigurationException();
 			}
-		} else if (GlobalConstants.tune == 0) {
+		} else if (Options.tune == 0) {
 			this.cfg = cfgFromFile(app, controller, defaultCfg);
 		} else
 			this.cfg = defaultCfg;
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index 2ceda404..d6f2aae3 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -38,6 +38,7 @@
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.concurrent.ConcurrentChannelFactory;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
@@ -155,7 +156,7 @@ public boolean newPartitionMap(
 	public void verifyConfiguration(
 			Map<Integer, List<Set<Worker<?, ?>>>> partitionsMachineMap) {
 
-		if (!GlobalConstants.singleNodeOnline) {
+		if (!Options.singleNodeOnline) {
 			// printPartition(partitionsMachineMap);
 		}
 
@@ -396,7 +397,7 @@ public Configuration getDynamicConfiguration() {
 		partParam.addBlobFactory(comp2Factory);
 		blobtoMachineMap = new HashMap<>();
 
-		BlobFactory bf = GlobalConstants.useCompilerBlob ? comp2Factory
+		BlobFactory bf = Options.useCompilerBlob ? comp2Factory
 				: intFactory;
 		for (Integer machineID : partitionsMachineMap.keySet()) {
 			List<Set<Worker<?, ?>>> blobList = partitionsMachineMap
@@ -412,7 +413,7 @@ public Configuration getDynamicConfiguration() {
 		}
 
 		builder.addParameter(partParam.build());
-		if (GlobalConstants.useCompilerBlob)
+		if (Options.useCompilerBlob)
 			builder.addSubconfiguration("blobConfigs", getConfiguration());
 		else
 			builder.addSubconfiguration("blobConfigs", getInterpreterConfg());
@@ -436,6 +437,6 @@ private int maxCores() {
 				IntParameter.class);
 		if (maxCoreParam != null)
 			return maxCoreParam.getValue();
-		return GlobalConstants.maxNumCores;
+		return Options.maxNumCores;
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 5e9a6bda..22752d88 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -33,6 +33,7 @@
 import edu.mit.streamjit.impl.distributed.common.Error.ErrorProcessor;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.Drained;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainProcessor;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
@@ -154,7 +155,7 @@ public void drainingFinished(boolean isFinal) {
 		}
 
 		if (tailChannel != null) {
-			if (GlobalConstants.useDrainData)
+			if (Options.useDrainData)
 				if (isFinal)
 					tailChannel.stop(DrainType.FINAL);
 				else
@@ -291,7 +292,7 @@ private void reset() {
 
 	private MasterProfiler setupProfiler() {
 		MasterProfiler p = null;
-		if (GlobalConstants.needProfiler) {
+		if (Options.needProfiler) {
 			p = new MasterProfiler(app.name);
 			controller.sendToAll(ProfilerCommand.START);
 		}
@@ -338,11 +339,11 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 			throw new IllegalArgumentException(
 					"No tail buffer in the passed bufferMap.");
 
-		int skipCount = Math.max(GlobalConstants.outputCount, multiplier * 5);
+		int skipCount = Math.max(Options.outputCount, multiplier * 5);
 		tailChannel = new TailChannels.BlockingTailChannel2(
 				bufferMap.get(tailToken), controller.getConProvider(),
 				tailconInfo, "tailChannel - " + tailToken.toString(), 0,
-				skipCount, GlobalConstants.outputCount, app.name);
+				skipCount, Options.outputCount, app.name);
 	}
 
 	/**
@@ -487,7 +488,7 @@ public void process(Drained drained) {
 
 		@Override
 		public void process(SNDrainedData snDrainedData) {
-			if (GlobalConstants.useDrainData)
+			if (Options.useDrainData)
 				drainer.newSNDrainData(snDrainedData);
 		}
 	}
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index b1fb9343..7e6e5954 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -15,7 +15,7 @@
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
 public class TailChannels {
@@ -82,11 +82,11 @@ private void writeInitialInfo(FileWriter writer) {
 			System.out.println(String.format(
 					"PerformanceLogger starts to log the time to"
 							+ " produce %d number of outputs",
-					GlobalConstants.outputCount));
+					Options.outputCount));
 
 			try {
 				writer.write(String.format("GlobalConstants.outputCount = %d",
-						GlobalConstants.outputCount));
+						Options.outputCount));
 				writer.write('\n');
 				writer.flush();
 			} catch (IOException e) {
@@ -120,7 +120,7 @@ private static class OutputCountPrinter {
 		}
 
 		private void printOutputCount() {
-			if (GlobalConstants.printOutputCountPeriod < 1)
+			if (Options.printOutputCountPeriod < 1)
 				return;
 
 			lastCount = 0;
@@ -137,12 +137,12 @@ public void run() {
 							System.out.println(String
 									.format("Outputs: since started - %d, during last %d ms - %d",
 											currentCount,
-											GlobalConstants.printOutputCountPeriod,
+											Options.printOutputCountPeriod,
 											newOutputs));
 
 						}
-					}, GlobalConstants.printOutputCountPeriod,
-					GlobalConstants.printOutputCountPeriod,
+					}, Options.printOutputCountPeriod,
+					Options.printOutputCountPeriod,
 					TimeUnit.MILLISECONDS);
 		}
 
@@ -192,13 +192,13 @@ public AbstractBlockingTailChannel(Buffer buffer,
 			this.skipCount = skipCount;
 			this.totalCount = steadyCount + skipCount;
 			count = 0;
-			if (GlobalConstants.tune == 0) {
+			if (Options.tune == 0) {
 				// TODO: Leaks this object from the constructor. May cause
 				// subtle bugs. Re-factor it.
 				pLogger = new PerformanceLogger(this, appName);
 				pLogger.start();
 			}
-			if (GlobalConstants.printOutputCountPeriod > 0)
+			if (Options.printOutputCountPeriod > 0)
 				outputCountPrinter = new OutputCountPrinter(this);
 		}
 
@@ -219,7 +219,7 @@ public int count() {
 		}
 
 		protected long normalizedTime(long time) {
-			return (GlobalConstants.outputCount * time)
+			return (Options.outputCount * time)
 					/ (totalCount - skipCount);
 		}
 
@@ -229,7 +229,7 @@ protected long normalizedTime(long time) {
 		 */
 		protected long unnormalizedTime(long time) {
 			return (time * (totalCount - skipCount))
-					/ GlobalConstants.outputCount;
+					/ Options.outputCount;
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
index 064f9fd2..1fc70dc6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/GlobalConstants.java
@@ -1,18 +1,7 @@
 package edu.mit.streamjit.impl.distributed.common;
 
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Properties;
-
-import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
-import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
-import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.StreamNodeAgent;
-import edu.mit.streamjit.tuner.TCPTuner;
 
 /**
  * This class is to keep track of all application level constants. So we can
@@ -66,172 +55,4 @@ private GlobalConstants() {
 	public static final String PORTID_MAP = "portIdMap";
 	public static final String PARTITION = "partition";
 	public static final String CONINFOMAP = "ConInfoMap";
-
-	/**
-	 * Decides how to start the opentuner. In first 2 cases, controller starts
-	 * opentuner and establishes connection with it on a random port no range
-	 * from 5000-65536. User can provide port no in 3 case.
-	 * 
-	 * <ol>
-	 * <li>0 - Controller starts the tuner automatically on a terminal. User can
-	 * see Opentuner related outputs in the new terminal.
-	 * <li>1 - Controller starts the tuner automatically as a Python process. No
-	 * explicit window will be opened. Suitable for remote running through SSH
-	 * terminal.
-	 * <li>2 - User has to manually start the tuner with correct portNo as
-	 * argument. Port no 12563 is used in this case. But it can be changed at
-	 * {@link TCPTuner#startTuner(String)}. We need this option to run the
-	 * tuning on remote machines.
-	 * </ol>
-	 */
-	public static final int tunerStartMode;
-
-	/**
-	 * To turn on or turn off the drain data. If this is false, drain data will
-	 * be ignored and every new reconfiguration will run with fresh inputs.
-	 */
-	public static final boolean useDrainData;
-
-	/**
-	 * To turn on or off the dead lock handler. see {@link AbstractDrainer} for
-	 * it's usage.
-	 */
-	public static final boolean needDrainDeadlockHandler;
-
-	/**
-	 * Enables tuning. Tuner will be started iff this flag is set true.
-	 * Otherwise, just use the fixed configuration file to run the program. No
-	 * tuning, no intermediate draining. In this mode (tune = false), time taken
-	 * to pass fixed number of input will be measured for 30 rounds and logged
-	 * into FixedOutPut.txt. See {@link TailChannels} for the file logging
-	 * details.
-	 * <ol>
-	 * 0 - No tuning, uses configuration file to run.
-	 * <ol>
-	 * 1 - Tuning.
-	 * <ol>
-	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
-	 * cfg. Both file should be presented in the running directory.
-	 */
-	public static final int tune;
-
-	/**
-	 * Save all configurations tired by open tuner in to
-	 * "configurations//app.name" directory.
-	 */
-	public static final boolean saveAllConfigurations;
-
-	/**
-	 * Output count for tuning. Tuner measures the running time for this number
-	 * of outputs.
-	 */
-	public static final int outputCount;
-
-	/**
-	 * if true uses Compiler2, interpreter otherwise.
-	 */
-	public static final boolean useCompilerBlob;
-
-	/**
-	 * Period to print output count periodically. This printing feature get
-	 * turned off if this value is less than 1. Time unit is ms. See
-	 * {@link TailChannels}.
-	 */
-	public static final int printOutputCountPeriod;
-
-	/**
-	 * Enables {@link DistributedStreamCompiler} to run on a single node. When
-	 * this is enabled, noOfNodes passed as compiler argument has no effect.
-	 */
-	public static final boolean singleNodeOnline;
-
-	/**
-	 * We can set this value at class loading time also as follows.
-	 * 
-	 * <code>maxThreadCount = Math.max(Runtime.getntime().availableProcessors() / 2,
-	 * 1);</code>
-	 * 
-	 * Lets hard code this for the moment.
-	 */
-	public static final int maxNumCores;
-
-	/**
-	 * Turn On/Off the profiling.
-	 */
-	public static final boolean needProfiler;
-
-	static {
-		Properties prop = loadProperties();
-		printOutputCountPeriod = Integer.parseInt(prop
-				.getProperty("printOutputCountPeriod"));;
-		maxNumCores = Integer.parseInt(prop.getProperty("maxNumCores"));
-		useCompilerBlob = Boolean.parseBoolean(prop
-				.getProperty("useCompilerBlob"));
-		needDrainDeadlockHandler = Boolean.parseBoolean(prop
-				.getProperty("needDrainDeadlockHandler"));
-		needProfiler = Boolean.parseBoolean(prop.getProperty("needProfiler"));
-		outputCount = Integer.parseInt(prop.getProperty("outputCount"));
-		tune = Integer.parseInt(prop.getProperty("tune"));
-		tunerStartMode = Integer.parseInt(prop.getProperty("tunerStartMode"));
-		saveAllConfigurations = Boolean.parseBoolean(prop
-				.getProperty("saveAllConfigurations"));
-		singleNodeOnline = Boolean.parseBoolean(prop
-				.getProperty("singleNodeOnline"));
-		useDrainData = Boolean.parseBoolean(prop.getProperty("useDrainData"));
-	}
-
-	private static Properties loadProperties() {
-		Properties prop = new Properties();
-		InputStream input = null;
-		try {
-			input = new FileInputStream("options.properties");
-			prop.load(input);
-		} catch (IOException ex) {
-			System.err.println("Failed to load options.properties");
-		}
-		return prop;
-	}
-
-	public static void storeProperties() {
-		OutputStream output = null;
-		try {
-			output = new FileOutputStream("options.properties");
-			Properties prop = getProperties();
-			prop.store(output, null);
-		} catch (IOException io) {
-			io.printStackTrace();
-		} finally {
-			if (output != null) {
-				try {
-					output.close();
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-	}
-
-	private static void setProperty(Properties prop, String name, Integer val) {
-		prop.setProperty(name, val.toString());
-	}
-
-	private static void setProperty(Properties prop, String name, Boolean val) {
-		prop.setProperty(name, val.toString());
-	}
-
-	public static Properties getProperties() {
-		Properties prop = new Properties();
-		setProperty(prop, "tunerStartMode", tunerStartMode);
-		setProperty(prop, "useDrainData", useDrainData);
-		setProperty(prop, "needDrainDeadlockHandler", needDrainDeadlockHandler);
-		setProperty(prop, "tune", tune);
-		setProperty(prop, "saveAllConfigurations", saveAllConfigurations);
-		setProperty(prop, "outputCount", outputCount);
-		setProperty(prop, "useCompilerBlob", useCompilerBlob);
-		setProperty(prop, "printOutputCountPeriod", printOutputCountPeriod);
-		setProperty(prop, "singleNodeOnline", singleNodeOnline);
-		setProperty(prop, "maxNumCores", maxNumCores);
-		setProperty(prop, "needProfiler", needProfiler);
-		return prop;
-	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 295f6389..79e5f6e6 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -140,7 +140,7 @@ public static void writeReadMeTxt(String appName) {
 			Calendar cal = Calendar.getInstance();
 			writer.write(dateFormat.format(cal.getTime()) + "\n");
 			writer.write(appName + "\n");
-			Properties prop = GlobalConstants.getProperties();
+			Properties prop = Options.getProperties();
 			prop.store(writer, "GlobalConstants.Properties");
 			writer.close();
 		} catch (IOException e) {
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b15db2c8..bf7620d6 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -19,7 +19,7 @@
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.OpenTuner;
@@ -62,9 +62,9 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 
 	@Override
 	public void run() {
-		if (GlobalConstants.tune == 1)
+		if (Options.tune == 1)
 			tune();
-		else if (GlobalConstants.tune == 2) {
+		else if (Options.tune == 2) {
 			verifier.verify(true);
 			terminate();
 		} else
@@ -250,7 +250,7 @@ private Configuration newCfg(int round, String cfgJson) {
 		Configuration config = Configuration.fromJson(cfgJson);
 		config = ConfigurationUtils.addConfigPrefix(config, cfgPrefix);
 
-		if (GlobalConstants.saveAllConfigurations)
+		if (Options.saveAllConfigurations)
 			ConfigurationUtils.saveConfg(cfgJson, cfgPrefix, app.name);
 		return config;
 	}
@@ -352,7 +352,7 @@ private FileWriter writer() throws IOException {
 			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
 					app.name, File.separator, app.name), true);
 			writer.write("##########################################################");
-			Properties prop = GlobalConstants.getProperties();
+			Properties prop = Options.getProperties();
 			prop.store(writer, "");
 			return writer;
 		}
diff --git a/src/edu/mit/streamjit/tuner/TCPTuner.java b/src/edu/mit/streamjit/tuner/TCPTuner.java
index a81588a8..961b23ad 100644
--- a/src/edu/mit/streamjit/tuner/TCPTuner.java
+++ b/src/edu/mit/streamjit/tuner/TCPTuner.java
@@ -12,7 +12,7 @@
 import java.net.UnknownHostException;
 import java.util.Random;
 
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 
 public final class TCPTuner implements OpenTuner {
 
@@ -54,14 +54,14 @@ public void startTuner(String tunerPath, File workingDir)
 		int min = 5000;
 		Random rand = new Random();
 		Integer port = rand.nextInt(65535 - min) + min;
-		if (GlobalConstants.tunerStartMode == 0) {
+		if (Options.tunerStartMode == 0) {
 			ProcessBuilder xtermPB = new ProcessBuilder("xterm", "-hold", "-e",
 					"python", tunerPath, port.toString());
 			ProcessBuilder gnomePB = new ProcessBuilder("gnome-terminal", "-e",
 					String.format("python %s %s", tunerPath, port.toString()));
 			gnomePB.directory(workingDir);
 			this.tuner = gnomePB.start();
-		} else if (GlobalConstants.tunerStartMode == 1) {
+		} else if (Options.tunerStartMode == 1) {
 			ProcessBuilder pb = new ProcessBuilder("python", tunerPath,
 					port.toString());
 			pb.directory(workingDir);

From 6d4aed801184d795c5372099073d71d64f331502 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Mar 2015 22:37:01 +0800
Subject: [PATCH 782/881] Options.java added.

---
 .../impl/distributed/common/Options.java      | 190 ++++++++++++++++++
 1 file changed, 190 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/common/Options.java

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
new file mode 100644
index 00000000..694f69c8
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -0,0 +1,190 @@
+package edu.mit.streamjit.impl.distributed.common;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Properties;
+
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
+import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
+import edu.mit.streamjit.impl.distributed.TailChannels;
+import edu.mit.streamjit.tuner.TCPTuner;
+
+/**
+ * Program options. Loads the values from "options.properties".
+ * 
+ * @author sumanan
+ * @since 1 Mar, 2015
+ */
+public final class Options {
+
+	/**
+	 * We can set this value at class loading time also as follows.
+	 * 
+	 * <code>maxThreadCount = Math.max(Runtime.getntime().availableProcessors() / 2,
+	 * 1);</code>
+	 * 
+	 * Lets hard code this for the moment.
+	 */
+	public static final int maxNumCores;
+
+	/**
+	 * To turn on or off the dead lock handler. see {@link AbstractDrainer} for
+	 * it's usage.
+	 */
+	public static final boolean needDrainDeadlockHandler;
+
+	/**
+	 * Turn On/Off the profiling.
+	 */
+	public static final boolean needProfiler;
+
+	/**
+	 * Output count for tuning. Tuner measures the running time for this number
+	 * of outputs.
+	 */
+	public static final int outputCount;
+
+	/**
+	 * Period to print output count periodically. This printing feature get
+	 * turned off if this value is less than 1. Time unit is ms. See
+	 * {@link TailChannels}.
+	 */
+	public static final int printOutputCountPeriod;
+
+	/**
+	 * Save all configurations tired by open tuner in to
+	 * "configurations//app.name" directory.
+	 */
+	public static final boolean saveAllConfigurations;
+
+	/**
+	 * Enables {@link DistributedStreamCompiler} to run on a single node. When
+	 * this is enabled, noOfNodes passed as compiler argument has no effect.
+	 */
+	public static final boolean singleNodeOnline;
+
+	/**
+	 * Enables tuning. Tuner will be started iff this flag is set true.
+	 * Otherwise, just use the fixed configuration file to run the program. No
+	 * tuning, no intermediate draining. In this mode (tune = false), time taken
+	 * to pass fixed number of input will be measured for 30 rounds and logged
+	 * into FixedOutPut.txt. See {@link TailChannels} for the file logging
+	 * details.
+	 * <ol>
+	 * 0 - No tuning, uses configuration file to run.
+	 * <ol>
+	 * 1 - Tuning.
+	 * <ol>
+	 * 2 - Evaluate configuration files. ( compares final cfg with hand tuned
+	 * cfg. Both file should be presented in the running directory.
+	 */
+	public static final int tune;
+
+	/**
+	 * Decides how to start the opentuner. In first 2 cases, controller starts
+	 * opentuner and establishes connection with it on a random port no range
+	 * from 5000-65536. User can provide port no in 3 case.
+	 * 
+	 * <ol>
+	 * <li>0 - Controller starts the tuner automatically on a terminal. User can
+	 * see Opentuner related outputs in the new terminal.
+	 * <li>1 - Controller starts the tuner automatically as a Python process. No
+	 * explicit window will be opened. Suitable for remote running through SSH
+	 * terminal.
+	 * <li>2 - User has to manually start the tuner with correct portNo as
+	 * argument. Port no 12563 is used in this case. But it can be changed at
+	 * {@link TCPTuner#startTuner(String)}. We need this option to run the
+	 * tuning on remote machines.
+	 * </ol>
+	 */
+	public static final int tunerStartMode;
+
+	/**
+	 * if true uses Compiler2, interpreter otherwise.
+	 */
+	public static final boolean useCompilerBlob;
+
+	/**
+	 * To turn on or turn off the drain data. If this is false, drain data will
+	 * be ignored and every new reconfiguration will run with fresh inputs.
+	 */
+	public static final boolean useDrainData;
+
+	static {
+		Properties prop = loadProperties();
+		printOutputCountPeriod = Integer.parseInt(prop
+				.getProperty("printOutputCountPeriod"));;
+		maxNumCores = Integer.parseInt(prop.getProperty("maxNumCores"));
+		useCompilerBlob = Boolean.parseBoolean(prop
+				.getProperty("useCompilerBlob"));
+		needDrainDeadlockHandler = Boolean.parseBoolean(prop
+				.getProperty("needDrainDeadlockHandler"));
+		needProfiler = Boolean.parseBoolean(prop.getProperty("needProfiler"));
+		outputCount = Integer.parseInt(prop.getProperty("outputCount"));
+		tune = Integer.parseInt(prop.getProperty("tune"));
+		tunerStartMode = Integer.parseInt(prop.getProperty("tunerStartMode"));
+		saveAllConfigurations = Boolean.parseBoolean(prop
+				.getProperty("saveAllConfigurations"));
+		singleNodeOnline = Boolean.parseBoolean(prop
+				.getProperty("singleNodeOnline"));
+		useDrainData = Boolean.parseBoolean(prop.getProperty("useDrainData"));
+	}
+
+	public static Properties getProperties() {
+		Properties prop = new Properties();
+		setProperty(prop, "tunerStartMode", tunerStartMode);
+		setProperty(prop, "useDrainData", useDrainData);
+		setProperty(prop, "needDrainDeadlockHandler", needDrainDeadlockHandler);
+		setProperty(prop, "tune", tune);
+		setProperty(prop, "saveAllConfigurations", saveAllConfigurations);
+		setProperty(prop, "outputCount", outputCount);
+		setProperty(prop, "useCompilerBlob", useCompilerBlob);
+		setProperty(prop, "printOutputCountPeriod", printOutputCountPeriod);
+		setProperty(prop, "singleNodeOnline", singleNodeOnline);
+		setProperty(prop, "maxNumCores", maxNumCores);
+		setProperty(prop, "needProfiler", needProfiler);
+		return prop;
+	}
+
+	private static Properties loadProperties() {
+		Properties prop = new Properties();
+		InputStream input = null;
+		try {
+			input = new FileInputStream("options.properties");
+			prop.load(input);
+		} catch (IOException ex) {
+			System.err.println("Failed to load options.properties");
+		}
+		return prop;
+	}
+
+	private static void setProperty(Properties prop, String name, Boolean val) {
+		prop.setProperty(name, val.toString());
+	}
+
+	private static void setProperty(Properties prop, String name, Integer val) {
+		prop.setProperty(name, val.toString());
+	}
+
+	public static void storeProperties() {
+		OutputStream output = null;
+		try {
+			output = new FileOutputStream("options.properties");
+			Properties prop = getProperties();
+			prop.store(output, null);
+		} catch (IOException io) {
+			io.printStackTrace();
+		} finally {
+			if (output != null) {
+				try {
+					output.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+	}
+}

From d7cb5cad21fb43425129cfeb5169ae47c9af3aa6 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Mar 2015 23:12:00 +0800
Subject: [PATCH 783/881] Options.connectionManager added

DistributedStreamCompiler creates ConnectionManager accordingly.
---
 .../DistributedStreamCompiler.java            | 17 ++++++++++++++--
 .../impl/distributed/common/Options.java      | 20 +++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 0d1e5d17..e113f340 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -119,8 +119,8 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		PartitionManager partitionManager = new HotSpotTuning(app);
 		ConfigurationManager cfgManager = new ConfigurationManager(app,
 				partitionManager);
-		ConnectionManager conManager = new ConnectionManager.AsyncTCPNoParams(
-				controller.controllerNodeID);
+		ConnectionManager conManager = connectionManager(controller.controllerNodeID);
+
 		setConfiguration(controller, app, partitionManager, conManager,
 				cfgManager);
 
@@ -144,6 +144,19 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		return cs;
 	}
 
+	private ConnectionManager connectionManager(int controllerNodeID) {
+		switch (Options.connectionManager) {
+			case 0 :
+				return new ConnectionManager.AllConnectionParams(
+						controllerNodeID);
+			case 1 :
+				return new ConnectionManager.BlockingTCPNoParams(
+						controllerNodeID);
+			default :
+				return new ConnectionManager.AsyncTCPNoParams(controllerNodeID);
+		}
+	}
+
 	private <I, O> Configuration cfgFromFile(StreamJitApp<I, O> app,
 			Controller controller, Configuration defaultCfg) {
 		Configuration cfg1 = ConfigurationUtils.readConfiguration(app.name,
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 694f69c8..89e67f3a 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -8,6 +8,9 @@
 import java.util.Properties;
 
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.AllConnectionParams;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.AsyncTCPNoParams;
+import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.tuner.TCPTuner;
@@ -113,6 +116,20 @@ public final class Options {
 	 */
 	public static final boolean useDrainData;
 
+	// Following are miscellaneous options to avoid rebuilding jar files every
+	// time to change some class selections.
+	// TODO: Fix all design pattern related issues.
+
+	/**
+	 * <ol>
+	 * <li>0 - {@link AllConnectionParams}
+	 * <li>1 - {@link BlockingTCPNoParams}
+	 * <li>2 - {@link AsyncTCPNoParams}
+	 * <li>default: {@link AsyncTCPNoParams}
+	 * </ol>
+	 */
+	public static final int connectionManager;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -131,6 +148,8 @@ public final class Options {
 		singleNodeOnline = Boolean.parseBoolean(prop
 				.getProperty("singleNodeOnline"));
 		useDrainData = Boolean.parseBoolean(prop.getProperty("useDrainData"));
+		connectionManager = Integer.parseInt(prop
+				.getProperty("connectionManager"));
 	}
 
 	public static Properties getProperties() {
@@ -146,6 +165,7 @@ public static Properties getProperties() {
 		setProperty(prop, "singleNodeOnline", singleNodeOnline);
 		setProperty(prop, "maxNumCores", maxNumCores);
 		setProperty(prop, "needProfiler", needProfiler);
+		setProperty(prop, "connectionManager", connectionManager);
 		return prop;
 	}
 

From a88029bd3242f8031dde0595c9df10f5e1a9a54a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 1 Mar 2015 23:26:45 +0800
Subject: [PATCH 784/881] Options.tailChannel added.

StreamJitAppManager creates tailChannel based on this option.
---
 .../impl/distributed/StreamJitAppManager.java | 25 ++++++++++++++++---
 .../impl/distributed/common/Options.java      | 13 ++++++++++
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 22752d88..8fc54bab 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -30,6 +30,7 @@
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString;
 import edu.mit.streamjit.impl.distributed.common.ConfigurationString.ConfigurationProcessor.ConfigType;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
+import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Error.ErrorProcessor;
 import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
 import edu.mit.streamjit.impl.distributed.common.MiscCtrlElements.NewConInfo;
@@ -340,10 +341,26 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(Options.outputCount, multiplier * 5);
-		tailChannel = new TailChannels.BlockingTailChannel2(
-				bufferMap.get(tailToken), controller.getConProvider(),
-				tailconInfo, "tailChannel - " + tailToken.toString(), 0,
-				skipCount, Options.outputCount, app.name);
+		tailChannel = tailChannel(bufferMap.get(tailToken),
+				controller.getConProvider(), tailconInfo, "tailChannel - "
+						+ tailToken.toString(), 0, skipCount,
+				Options.outputCount, app.name);
+	}
+
+	private TailChannel tailChannel(Buffer buffer,
+			ConnectionProvider conProvider, ConnectionInfo conInfo,
+			String bufferTokenName, int debugLevel, int skipCount,
+			int steadyCount, String appName) {
+		switch (Options.tailChannel) {
+			case 1 :
+				return new TailChannels.BlockingTailChannel1(buffer,
+						conProvider, conInfo, bufferTokenName, debugLevel,
+						skipCount, steadyCount, appName);
+			default :
+				return new TailChannels.BlockingTailChannel2(buffer,
+						conProvider, conInfo, bufferTokenName, debugLevel,
+						skipCount, steadyCount, appName);
+		}
 	}
 
 	/**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 89e67f3a..2ca2a7cf 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -13,6 +13,8 @@
 import edu.mit.streamjit.impl.distributed.ConnectionManager.BlockingTCPNoParams;
 import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.distributed.TailChannels;
+import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel1;
+import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel2;
 import edu.mit.streamjit.tuner.TCPTuner;
 
 /**
@@ -130,6 +132,15 @@ public final class Options {
 	 */
 	public static final int connectionManager;
 
+	/**
+	 * <ol>
+	 * <li>1 - {@link BlockingTailChannel1}
+	 * <li>2 - {@link BlockingTailChannel2}
+	 * <li>default: {@link BlockingTailChannel2}
+	 * </ol>
+	 */
+	public static final int tailChannel;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -150,6 +161,7 @@ public final class Options {
 		useDrainData = Boolean.parseBoolean(prop.getProperty("useDrainData"));
 		connectionManager = Integer.parseInt(prop
 				.getProperty("connectionManager"));
+		tailChannel = Integer.parseInt(prop.getProperty("tailChannel"));
 	}
 
 	public static Properties getProperties() {
@@ -166,6 +178,7 @@ public static Properties getProperties() {
 		setProperty(prop, "maxNumCores", maxNumCores);
 		setProperty(prop, "needProfiler", needProfiler);
 		setProperty(prop, "connectionManager", connectionManager);
+		setProperty(prop, "tailChannel", tailChannel);
 		return prop;
 	}
 

From b69b7e061e4a2cb4f5b776d2cfeeb97430fbb695 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 14:12:45 +0800
Subject: [PATCH 785/881] Uses jdk1.8.0_31

---
 scripts/setup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/setup.sh b/scripts/setup.sh
index 07aeb15a..c25b97f6 100644
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -17,7 +17,7 @@ function writeRun(){
 		echo "srun python ../lib/opentuner/streamjit/streamjit2.py 12563 &" >> $runfile
 		echo "cd .." >> $runfile
 	fi
-	echo "srun -l ../bin/java/jdk1.8.0_25/bin/java -Xmx2048m -jar $1.jar $3" >> $runfile
+	echo "srun -l ../bin/java/jdk1.8.0_31/bin/java -Xmx2048m -jar $1.jar $3" >> $runfile
 }
 
 function writeSN(){
@@ -28,7 +28,7 @@ function writeSN(){
 	echo "#SBATCH --cpu_bind=verbose,cores" >> $runfile
 	echo "#SBATCH --exclusive" >> $runfile
 	echo "cd /data/scratch/sumanan/"$1 >> $runfile
-	echo "srun --exclusive  --nodes=$2 ../bin/java/jdk1.8.0_25/bin/java -Xmx2048m -jar StreamNode.jar 128.30.116." >> $runfile
+	echo "srun --exclusive  --nodes=$2 ../bin/java/jdk1.8.0_31/bin/java -Xmx2048m -jar StreamNode.jar 128.30.116." >> $runfile
 }
 
 function creatdirs(){

From a962519a2fa5b64702b6b39662e95f9b4ba55f35 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 15:43:05 +0800
Subject: [PATCH 786/881] File path problem in writeHeapStat() corrected.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 9449623f..b74c8a4f 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -137,7 +137,9 @@ private static void writeHeapStat(String fileName, File outDir)
 		List<Integer> heapSize = processSNHeap(fileName, false);
 		List<Integer> heapMaxSize = processSNHeap(fileName, true);
 
-		String outFileName = String.format("%s_heapStatus.txt", fileName);
+		File f = new File(fileName);
+
+		String outFileName = String.format("%s_heapStatus.txt", f.getName());
 		File outFile = new File(outDir, outFileName);
 		FileWriter writer = new FileWriter(outFile, false);
 		for (int i = 0; i < heapSize.size(); i++) {

From e46a904ed7d96dae7440b2dd7535a6ce6a927f1c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 15:49:56 +0800
Subject: [PATCH 787/881] New method to summarize heap status added.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index b74c8a4f..350e2dd5 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -215,9 +215,6 @@ public static void summarize(String appName) throws IOException {
 
 		makePlotFile(summaryDir, appName, dataFile);
 		plot(summaryDir);
-
-		// writeHeapStat(String.format("%s%sst1.txt", appName, File.separator));
-		// writeHeapStat(String.format("%s%sst2.txt", appName, File.separator));
 	}
 
 	private static void makePlotFile(File dir, String name, String dataFile)
@@ -271,4 +268,16 @@ private static void plot(File dir) throws IOException {
 			System.err.println("Fail: " + e);
 		}
 	}
+
+	public static void summarizeHeap(String appName) throws IOException {
+		File summaryDir = new File(String.format("%s%ssummary", appName,
+				File.separator));
+		Utils.createDir(summaryDir.getPath());
+		writeHeapStat(
+				String.format("%s%sslurm-662553.out", appName, File.separator),
+				summaryDir);
+		writeHeapStat(
+				String.format("%s%sslurm-662554.out", appName, File.separator),
+				summaryDir);
+	}
 }

From f1cb7e5c6fcefc4e4962275f3b467bdda9838ace Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 15:54:19 +0800
Subject: [PATCH 788/881] makeHeapPlotFile() added.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 34 ++++++++++++++++---
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 350e2dd5..29c55b4c 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -132,13 +132,11 @@ private static Map<String, Integer> process(BufferedReader reader,
 		return ret;
 	}
 
-	private static void writeHeapStat(String fileName, File outDir)
+	private static File writeHeapStat(String fileName, File outDir)
 			throws IOException {
 		List<Integer> heapSize = processSNHeap(fileName, false);
 		List<Integer> heapMaxSize = processSNHeap(fileName, true);
-
 		File f = new File(fileName);
-
 		String outFileName = String.format("%s_heapStatus.txt", f.getName());
 		File outFile = new File(outDir, outFileName);
 		FileWriter writer = new FileWriter(outFile, false);
@@ -148,6 +146,7 @@ private static void writeHeapStat(String fileName, File outDir)
 			writer.write(msg);
 		}
 		writer.close();
+		return outFile;
 	}
 
 	private static List<Integer> processSNHeap(String fileName,
@@ -246,6 +245,30 @@ private static void makePlotFile(File dir, String name, String dataFile)
 		writer.close();
 	}
 
+	private static void makeHeapPlotFile(File dir, String name,
+			String dataFile1, String dataFile2) throws IOException {
+		File plotfile = new File(dir, "heapplot.plt");
+		FileWriter writer = new FileWriter(plotfile, false);
+		writer.write("set terminal postscript eps enhanced color\n");
+		writer.write(String.format("set output \"%sHeap.eps\"\n", name));
+		writer.write("set ylabel \"Memory(MB)\"\n");
+		writer.write("set xlabel \"Tuning Rounds\"\n");
+		writer.write(String.format("set title \"%sHeap\"\n", name));
+		writer.write("set grid\n");
+		writer.write("#set yrange [0:*]\n");
+
+		writer.write(String
+				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
+						+ " plot \"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
+						dataFile1, dataFile1));
+		writer.write(String
+				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
+						+ " plot \"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
+						dataFile2, dataFile2));
+
+		writer.close();
+	}
+
 	private static void plot(File dir) throws IOException {
 		String[] s = { "/usr/bin/gnuplot", "plot.plt" };
 		try {
@@ -273,11 +296,12 @@ public static void summarizeHeap(String appName) throws IOException {
 		File summaryDir = new File(String.format("%s%ssummary", appName,
 				File.separator));
 		Utils.createDir(summaryDir.getPath());
-		writeHeapStat(
+		File f1 = writeHeapStat(
 				String.format("%s%sslurm-662553.out", appName, File.separator),
 				summaryDir);
-		writeHeapStat(
+		File f2 = writeHeapStat(
 				String.format("%s%sslurm-662554.out", appName, File.separator),
 				summaryDir);
+		makeHeapPlotFile(summaryDir, appName, f1.getName(), f2.getName());
 	}
 }

From ca279069b8cc0fba8280eaa4ef023c23f2890799 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 15:56:38 +0800
Subject: [PATCH 789/881] makeplotfile() returns the plotFile.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 29c55b4c..49274233 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -216,7 +216,7 @@ public static void summarize(String appName) throws IOException {
 		plot(summaryDir);
 	}
 
-	private static void makePlotFile(File dir, String name, String dataFile)
+	private static File makePlotFile(File dir, String name, String dataFile)
 			throws IOException {
 		File plotfile = new File(dir, "plot.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
@@ -243,9 +243,10 @@ private static void makePlotFile(File dir, String name, String dataFile)
 				.format("plot \"%s\" using 1:2 with linespoints title \"Tuning Round time\"\n",
 						dataFile));
 		writer.close();
+		return plotfile;
 	}
 
-	private static void makeHeapPlotFile(File dir, String name,
+	private static File makeHeapPlotFile(File dir, String name,
 			String dataFile1, String dataFile2) throws IOException {
 		File plotfile = new File(dir, "heapplot.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
@@ -267,6 +268,7 @@ private static void makeHeapPlotFile(File dir, String name,
 						dataFile2, dataFile2));
 
 		writer.close();
+		return plotfile;
 	}
 
 	private static void plot(File dir) throws IOException {

From e443b90dadd7b385f75c8e5bd5e48186d29c0a3e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 15:58:48 +0800
Subject: [PATCH 790/881] plot() takes plot file as arg.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 49274233..df71108f 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -212,8 +212,8 @@ public static void summarize(String appName) throws IOException {
 		verify.close();
 		writer.close();
 
-		makePlotFile(summaryDir, appName, dataFile);
-		plot(summaryDir);
+		File f = makePlotFile(summaryDir, appName, dataFile);
+		plot(summaryDir, f);
 	}
 
 	private static File makePlotFile(File dir, String name, String dataFile)
@@ -271,8 +271,8 @@ private static File makeHeapPlotFile(File dir, String name,
 		return plotfile;
 	}
 
-	private static void plot(File dir) throws IOException {
-		String[] s = { "/usr/bin/gnuplot", "plot.plt" };
+	private static void plot(File dir, File plotFile) throws IOException {
+		String[] s = { "/usr/bin/gnuplot", plotFile.getName() };
 		try {
 			ProcessBuilder pb = new ProcessBuilder(s);
 			pb.directory(dir);

From 3eb410cb738bdc22b22464b10da9646a6daf945c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 16:00:20 +0800
Subject: [PATCH 791/881] summarizeHeap calls plot() to generate the graph.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index df71108f..2a7335a1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -304,6 +304,8 @@ public static void summarizeHeap(String appName) throws IOException {
 		File f2 = writeHeapStat(
 				String.format("%s%sslurm-662554.out", appName, File.separator),
 				summaryDir);
-		makeHeapPlotFile(summaryDir, appName, f1.getName(), f2.getName());
+		File f = makeHeapPlotFile(summaryDir, appName, f1.getName(),
+				f2.getName());
+		plot(summaryDir, f);
 	}
 }

From 2853fca8812eada6b3e02a03a0b53cc114e94d39 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 16:06:41 +0800
Subject: [PATCH 792/881] Problem in heapplot.plt corrected.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 2a7335a1..b01684d1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -260,11 +260,11 @@ private static File makeHeapPlotFile(File dir, String name,
 
 		writer.write(String
 				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
-						+ " plot \"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
+						+ "\"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
 						dataFile1, dataFile1));
 		writer.write(String
 				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
-						+ " plot \"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
+						+ "\"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
 						dataFile2, dataFile2));
 
 		writer.close();

From 6b66d52eb2a49094eee79c72d2805b0a43ef931c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 16:12:44 +0800
Subject: [PATCH 793/881] TimeLogProcessor's main calls summarizeHeap()

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index b01684d1..6bd13533 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -24,6 +24,7 @@ public class TimeLogProcessor {
 
 	public static void main(String[] args) throws IOException {
 		summarize("FMRadioCore");
+		summarizeHeap("FMRadioCore");
 	}
 
 	private static Map<String, Integer> processCompileTime(String appName,

From 9273ffc43f1d405b456d290d2fe0889f299388a2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 2 Mar 2015 20:36:12 +0800
Subject: [PATCH 794/881] Processes GraphProperty.txt and prints cycle count.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 6bd13533..a7666d24 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -309,4 +309,27 @@ public static void summarizeHeap(String appName) throws IOException {
 				f2.getName());
 		plot(summaryDir, f);
 	}
+
+	public static void GraphPropertyProcessor(String appName)
+			throws IOException {
+		BufferedReader reader = new BufferedReader(
+				new FileReader(String.format("%s%sGraphProperty.txt", appName,
+						File.separator)));
+		int truecnt = 0;
+		int falsecnt = 0;
+		String line;
+		while ((line = reader.readLine()) != null) {
+			if (line.contains("True"))
+				truecnt++;
+			else if (line.contains("False"))
+				falsecnt++;
+			else
+				System.err.println("Not a tuning line...");
+		}
+		reader.close();
+		double total = truecnt + falsecnt;
+		System.out.println(String.format(
+				"Total=%f, TrueCount=%d(%f), FalseCount=%d(%f)", total,
+				truecnt, (truecnt / total), falsecnt, (falsecnt / total)));
+	}
 }

From 6a44becaf6a977ba774a1a1e7542010e7b7178a7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 07:20:57 +0800
Subject: [PATCH 795/881] Proceses HeadFreeSize and plots on the heap graph.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index a7666d24..1919e916 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -135,27 +135,25 @@ private static Map<String, Integer> process(BufferedReader reader,
 
 	private static File writeHeapStat(String fileName, File outDir)
 			throws IOException {
-		List<Integer> heapSize = processSNHeap(fileName, false);
-		List<Integer> heapMaxSize = processSNHeap(fileName, true);
+		List<Integer> heapSize = processSNHeap(fileName, "heapSize");
+		List<Integer> heapMaxSize = processSNHeap(fileName, "heapMaxSize");
+		List<Integer> heapFreeSize = processSNHeap(fileName, "heapFreeSize");
 		File f = new File(fileName);
 		String outFileName = String.format("%s_heapStatus.txt", f.getName());
 		File outFile = new File(outDir, outFileName);
 		FileWriter writer = new FileWriter(outFile, false);
 		for (int i = 0; i < heapSize.size(); i++) {
-			String msg = String.format("%-6d\t%-6d\t%-6d\n", i + 1,
-					heapSize.get(i), heapMaxSize.get(i));
+			String msg = String.format("%-6d\t%-6d\t%-6d\t%-6d\n", i + 1,
+					heapFreeSize.get(i), heapSize.get(i), heapMaxSize.get(i));
 			writer.write(msg);
 		}
 		writer.close();
 		return outFile;
 	}
 
-	private static List<Integer> processSNHeap(String fileName,
-			Boolean isHeapMax) throws IOException {
+	private static List<Integer> processSNHeap(String fileName, String heapType)
+			throws IOException {
 		String slurmPrefix = "0: ";
-		String heapType = "heapSize";
-		if (isHeapMax)
-			heapType = "heapMaxSize";
 		BufferedReader reader = new BufferedReader(new FileReader(fileName));
 		String line;
 		int i = 0;
@@ -260,13 +258,15 @@ private static File makeHeapPlotFile(File dir, String name,
 		writer.write("#set yrange [0:*]\n");
 
 		writer.write(String
-				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
-						+ "\"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
-						dataFile1, dataFile1));
+				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Free Size\","
+						+ "\"%s\" using 1:3 with linespoints title \"Heap Size\","
+						+ "\"%s\" using 1:4 with linespoints title \"Heap Max Size\" \n",
+						dataFile1, dataFile1, dataFile1));
 		writer.write(String
-				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Size\","
-						+ "\"%s\" using 1:3 with linespoints title \"Heap Max Size\" \n",
-						dataFile2, dataFile2));
+				.format("plot \"%s\" using 1:2 with linespoints title \"Heap Free Size\","
+						+ "\"%s\" using 1:3 with linespoints title \"Heap Size\","
+						+ "\"%s\" using 1:4 with linespoints title \"Heap Max Size\" \n",
+						dataFile2, dataFile2, dataFile2));
 
 		writer.close();
 		return plotfile;

From 319a3a45f92159f1e774dca946c65e423bfd335c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 07:58:34 +0800
Subject: [PATCH 796/881] Alwys backups the files generated during prev run.

---
 .../impl/distributed/StreamJitApp.java        |  3 +--
 .../impl/distributed/common/Utils.java        | 18 ++++++++++++++++
 .../distributed/runtimer/OnlineTuner.java     | 21 +++----------------
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index d6f2aae3..c52459f0 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -113,8 +113,7 @@ public StreamJitApp(OneToOneElement<I, O> streamGraph) {
 		this.jarFilePath = this.getClass().getProtectionDomain()
 				.getCodeSource().getLocation().getPath();
 		this.constraints = getConstrains();
-		Utils.createAppDir(name);
-		Utils.writeReadMeTxt(name);
+		Utils.newApp(name);
 		visualizer = new Visualizer.DotVisualizer(streamGraph);
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 79e5f6e6..17bb0adc 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -234,4 +234,22 @@ public static void generateBlobGraphs(OneToOneElement<?, ?> stream)
 	public static void main(String[] args) throws IOException {
 		generateBlobGraphs(new FMRadio.FMRadioCore());
 	}
+
+	/**
+	 * Backups the files generated during tuning.
+	 */
+	public static void backup(String appName) {
+		rename(appName, "summary");
+		rename(appName, "compileTime.txt");
+		rename(appName, "runTime.txt");
+		rename(appName, "drainTime.txt");
+		rename(appName, "GraphProperty.txt");
+		rename(appName, "profile.txt");
+	}
+
+	public static void newApp(String appName) {
+		createAppDir(appName);
+		backup(appName);
+		Utils.writeReadMeTxt(appName);
+	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index bf7620d6..2f214388 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -20,7 +20,6 @@
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Options;
-import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
@@ -65,7 +64,7 @@ public void run() {
 		if (Options.tune == 1)
 			tune();
 		else if (Options.tune == 2) {
-			verifier.verify(true);
+			verifier.verify();
 			terminate();
 		} else
 			System.err
@@ -228,7 +227,7 @@ private void handleTermination() throws IOException {
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
 
-		verifier.verify(false);
+		verifier.verify();
 
 		if (needTermination) {
 			terminate();
@@ -283,9 +282,7 @@ private void terminate() {
 
 	private class Verifier {
 
-		public void verify(boolean needBackup) {
-			if (needBackup)
-				backup();
+		public void verify() {
 			verifyTuningTimes(cfgPrefixes());
 		}
 
@@ -434,17 +431,5 @@ private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			}
 			return runningTime;
 		}
-
-		/**
-		 * Backups the files generated during tuning.
-		 */
-		private void backup() {
-			Utils.rename(app.name, "summary");
-			Utils.rename(app.name, "compileTime.txt");
-			Utils.rename(app.name, "runTime.txt");
-			Utils.rename(app.name, "drainTime.txt");
-			Utils.rename(app.name, "GraphProperty.txt");
-			Utils.rename(app.name, "profile.txt");
-		}
 	}
 }
\ No newline at end of file

From 5cbd1113854d14d4deff4ac90b87fe1668469c24 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 15:00:11 +0800
Subject: [PATCH 797/881] Options.evaluationCount added.

---
 .../mit/streamjit/impl/distributed/common/Options.java | 10 ++++++++++
 .../impl/distributed/runtimer/OnlineTuner.java         |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 2ca2a7cf..462a14ef 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -15,6 +15,7 @@
 import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel1;
 import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel2;
+import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
 
 /**
@@ -141,6 +142,13 @@ public final class Options {
 	 */
 	public static final int tailChannel;
 
+	/**
+	 * {@link OnlineTuner}'s verifier verifies the configurations if
+	 * {@link #tune}==2. evaluationCount determines the number of re runs for a
+	 * configuration.
+	 */
+	public static final int evaluationCount;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -162,6 +170,7 @@ public final class Options {
 		connectionManager = Integer.parseInt(prop
 				.getProperty("connectionManager"));
 		tailChannel = Integer.parseInt(prop.getProperty("tailChannel"));
+		evaluationCount = Integer.parseInt(prop.getProperty("evaluationCount"));
 	}
 
 	public static Properties getProperties() {
@@ -179,6 +188,7 @@ public static Properties getProperties() {
 		setProperty(prop, "needProfiler", needProfiler);
 		setProperty(prop, "connectionManager", connectionManager);
 		setProperty(prop, "tailChannel", tailChannel);
+		setProperty(prop, "evaluationCount", evaluationCount);
 		return prop;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2f214388..e969889a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -412,7 +412,7 @@ private void process2(String line, Map<String, Integer> cfgPrefixes) {
 		 */
 		private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			System.out.println("Evaluating " + cfgName);
-			int count = 2;
+			int count = Options.evaluationCount;
 			List<Long> runningTime = new ArrayList<>(count);
 			Pair<Boolean, Long> ret;
 			if (cfg != null) {

From f6bc356077c13500b38d78698eabe0e5773f37c5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 15:18:22 +0800
Subject: [PATCH 798/881] Options.verificationCount added.

---
 .../impl/distributed/common/Options.java      | 15 ++++++--
 .../distributed/runtimer/OnlineTuner.java     | 36 ++++++++++---------
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 462a14ef..85496288 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -120,7 +120,8 @@ public final class Options {
 	public static final boolean useDrainData;
 
 	// Following are miscellaneous options to avoid rebuilding jar files every
-	// time to change some class selections.
+	// time to change some class selections. You may decide to remove these
+	// variables in a stable release.
 	// TODO: Fix all design pattern related issues.
 
 	/**
@@ -145,10 +146,17 @@ public final class Options {
 	/**
 	 * {@link OnlineTuner}'s verifier verifies the configurations if
 	 * {@link #tune}==2. evaluationCount determines the number of re runs for a
-	 * configuration.
+	 * configuration. Default value is 2.
 	 */
 	public static final int evaluationCount;
 
+	/**
+	 * {@link OnlineTuner}'s verifier verifies the configurations if
+	 * {@link #tune}==2. verificationCount determines the number of re runs for
+	 * a set of configurations in the verify.txt. Default value is 1.
+	 */
+	public static final int verificationCount;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -171,6 +179,8 @@ public final class Options {
 				.getProperty("connectionManager"));
 		tailChannel = Integer.parseInt(prop.getProperty("tailChannel"));
 		evaluationCount = Integer.parseInt(prop.getProperty("evaluationCount"));
+		verificationCount = Integer.parseInt(prop
+				.getProperty("verificationCount"));
 	}
 
 	public static Properties getProperties() {
@@ -189,6 +199,7 @@ public static Properties getProperties() {
 		setProperty(prop, "connectionManager", connectionManager);
 		setProperty(prop, "tailChannel", tailChannel);
 		setProperty(prop, "evaluationCount", evaluationCount);
+		setProperty(prop, "verificationCount", verificationCount);
 		return prop;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index e969889a..14c90524 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -298,25 +298,27 @@ public void verify() {
 		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 			try {
 				FileWriter writer = writer();
-				for (Map.Entry<String, Integer> en : cfgPrefixes.entrySet()) {
-					String prefix = en.getKey();
-					Integer expectedRunningTime = en.getValue();
-					String cfgName = String.format("%s_%s.cfg", prefix,
-							app.name);
-					Configuration cfg = ConfigurationUtils.readConfiguration(
-							app.name, prefix);
-					if (cfg == null) {
-						System.err.println(String.format("No %s file exists",
+				for (int i = 0; i < Options.verificationCount; i++) {
+					for (Map.Entry<String, Integer> en : cfgPrefixes.entrySet()) {
+						String prefix = en.getKey();
+						Integer expectedRunningTime = en.getValue();
+						String cfgName = String.format("%s_%s.cfg", prefix,
+								app.name);
+						Configuration cfg = ConfigurationUtils
+								.readConfiguration(app.name, prefix);
+						if (cfg == null) {
+							System.err.println(String.format(
+									"No %s file exists", cfgName));
+							continue;
+						}
+						cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+						writer.write("----------------------------------------\n");
+						writer.write(String.format("Configuration name = %s\n",
 								cfgName));
-						continue;
+						List<Long> runningTimes = evaluateConfig(cfg, cfgName);
+						processRunningTimes(runningTimes, expectedRunningTime,
+								writer);
 					}
-					cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
-					writer.write("----------------------------------------\n");
-					writer.write(String.format("Configuration name = %s\n",
-							cfgName));
-					List<Long> runningTimes = evaluateConfig(cfg, cfgName);
-					processRunningTimes(runningTimes, expectedRunningTime,
-							writer);
 				}
 				writer.write("**************FINISHED**************\n\n");
 				writer.close();

From 5623ad375b5e063513c95dc10231a9d45af1a78e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 22:51:46 +0800
Subject: [PATCH 799/881] TimeLogger updates TuningRoundTime correctly

We shouldn't update the tuningRoundTime when starting the app.
---
 src/edu/mit/streamjit/impl/distributed/TimeLoggers.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 4a5bd6a5..b5fdd509 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -260,9 +260,9 @@ private void updateTuningRoundTime() {
 				time = tuningRoundSW.elapsed(TimeUnit.SECONDS);
 				tuningRoundSW.reset();
 				tuningRoundSW.start();
+				write(runTimeWriter,
+						String.format("Tuning round time %dS\n", time));
 			}
-			write(runTimeWriter,
-					String.format("Tuning round time %dS\n", time));
 		}
 
 		private static OutputStreamWriter getOSWriter(OutputStream os) {

From 0e189923abf7a3973dfdb10ed3cb348adfedbf4d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 3 Mar 2015 23:45:13 +0800
Subject: [PATCH 800/881] Options.multiplierMaxValue added.

Large multiplier -> Large compilation time and Large waiting time.
---
 .../streamjit/impl/compiler2/Compiler2BlobFactory.java    | 4 +++-
 .../mit/streamjit/impl/distributed/common/Options.java    | 8 ++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java b/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
index 959edf15..d3341bb7 100644
--- a/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
+++ b/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
@@ -6,7 +6,9 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.util.affinity.Affinity;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -58,7 +60,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		//add it here or we won't pick it up in the default configuration.
 //		Configuration.PermutationParameter<Integer> affinity = new Configuration.PermutationParameter<>("$affinity", Integer.class, Affinity.getMaximalAffinity());
 //		builder.addParameter(affinity);
-		return builder.addParameter(new Configuration.IntParameter("multiplier", 1, Short.MAX_VALUE, 1))
+		return builder.addParameter(new Configuration.IntParameter("multiplier", 1, Options.multiplierMaxValue, 1))
 				.build();
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 85496288..0a8417b8 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -157,6 +157,11 @@ public final class Options {
 	 */
 	public static final int verificationCount;
 
+	/**
+	 * Large multiplier -> Large compilation time and Large waiting time.
+	 */
+	public static final int multiplierMaxValue;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -181,6 +186,8 @@ public final class Options {
 		evaluationCount = Integer.parseInt(prop.getProperty("evaluationCount"));
 		verificationCount = Integer.parseInt(prop
 				.getProperty("verificationCount"));
+		multiplierMaxValue = Integer.parseInt(prop
+				.getProperty("multiplierMaxValue"));
 	}
 
 	public static Properties getProperties() {
@@ -200,6 +207,7 @@ public static Properties getProperties() {
 		setProperty(prop, "tailChannel", tailChannel);
 		setProperty(prop, "evaluationCount", evaluationCount);
 		setProperty(prop, "verificationCount", verificationCount);
+		setProperty(prop, "multiplierMaxValue", multiplierMaxValue);
 		return prop;
 	}
 

From 65179e05f0ca0b3a50640113da066606b776a0a0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 09:34:46 +0800
Subject: [PATCH 801/881] GraphProperty.txt: Tailing tab space removed.

tailing tab space at every line in GraphProperty.txt has been removed.
---
 .../distributed/runtimer/GraphPropertyPrognosticator.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 97d0ab03..8382d899 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -56,6 +56,7 @@ public boolean prognosticate(Configuration config) {
 		} catch (Exception e) {
 			e.printStackTrace();
 		}
+
 		return true;
 	}
 
@@ -157,7 +158,7 @@ private static void writeHeader(FileWriter writer) {
 			writer.write(String.format("%.7s", "hasCycles"));
 			writer.write("\t\t");
 			writer.write(String.format("%.7s", "time"));
-			writer.write("\t\t");
+			// writer.write("\t\t");
 			writer.flush();
 		} catch (IOException e) {
 
@@ -167,7 +168,7 @@ private static void writeHeader(FileWriter writer) {
 	@Override
 	public void time(double time) {
 		try {
-			writer.write(String.format("%.0f\t\t", time));
+			writer.write(String.format("%.0f", time));
 			writer.flush();
 		} catch (IOException e) {
 			e.printStackTrace();

From 63bfc05594294b6f8ce3843a745a533e17986422 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 11:04:09 +0800
Subject: [PATCH 802/881] GraphPropertyPrognosticator accept or reject

GraphPropertyPrognosticator decides a configuration based on the threshold
values in the Options.
---
 .../impl/distributed/common/Options.java      | 23 +++++++++
 .../runtimer/GraphPropertyPrognosticator.java | 48 +++++++++++++++++--
 2 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index 0a8417b8..e358d163 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -162,6 +162,16 @@ public final class Options {
 	 */
 	public static final int multiplierMaxValue;
 
+	public static final boolean prognosticate;
+
+	public static final int bigToSmallBlobRatio;
+
+	public static final int loadRatio;
+
+	public static final int blobToNodeRatio;
+
+	public static final int boundaryChannelRatio;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -188,6 +198,14 @@ public final class Options {
 				.getProperty("verificationCount"));
 		multiplierMaxValue = Integer.parseInt(prop
 				.getProperty("multiplierMaxValue"));
+		prognosticate = Boolean.parseBoolean(prop.getProperty("prognosticate"));
+		bigToSmallBlobRatio = Integer.parseInt(prop
+				.getProperty("bigToSmallBlobRatio"));
+		loadRatio = Integer.parseInt(prop.getProperty("loadRatio"));
+		blobToNodeRatio = Integer.parseInt(prop.getProperty("blobToNodeRatio"));
+		boundaryChannelRatio = Integer.parseInt(prop
+				.getProperty("boundaryChannelRatio"));
+
 	}
 
 	public static Properties getProperties() {
@@ -208,6 +226,11 @@ public static Properties getProperties() {
 		setProperty(prop, "evaluationCount", evaluationCount);
 		setProperty(prop, "verificationCount", verificationCount);
 		setProperty(prop, "multiplierMaxValue", multiplierMaxValue);
+		setProperty(prop, "prognosticate", prognosticate);
+		setProperty(prop, "bigToSmallBlobRatio", bigToSmallBlobRatio);
+		setProperty(prop, "loadRatio", loadRatio);
+		setProperty(prop, "blobToNodeRatio", blobToNodeRatio);
+		setProperty(prop, "boundaryChannelRatio", boundaryChannelRatio);
 		return prop;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 8382d899..c71dddf0 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -13,6 +13,7 @@
 import edu.mit.streamjit.impl.common.Workers;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.BoundaryChannel;
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
@@ -44,20 +45,59 @@ public boolean prognosticate(Configuration config) {
 		float bigToSmallBlobRatio = bigToSmallBlobRatio();
 		float loadRatio = loadRatio();
 		float blobToNodeRatio = blobToNodeRatio();
-		float BoundaryChannelRatio = totalToBoundaryChannelRatio();
+		float boundaryChannelRatio = totalToBoundaryChannelRatio();
 		boolean hasCycle = hasCycle();
 		try {
 			writer.write(String.format("\n%6s\t\t", cfgPrefix));
 			writer.write(String.format("%.2f\t\t", bigToSmallBlobRatio));
 			writer.write(String.format("%.2f\t\t", loadRatio));
 			writer.write(String.format("%.2f\t\t", blobToNodeRatio));
-			writer.write(String.format("%.2f\t\t", BoundaryChannelRatio));
+			writer.write(String.format("%.2f\t\t", boundaryChannelRatio));
 			writer.write(String.format("%s\t\t", hasCycle ? "True" : "False"));
 		} catch (Exception e) {
 			e.printStackTrace();
 		}
+		return decide(bigToSmallBlobRatio, loadRatio, blobToNodeRatio,
+				boundaryChannelRatio, hasCycle);
+	}
 
-		return true;
+	private boolean decide(float bigToSmallBlobRatio, float loadRatio,
+			float blobToNodeRatio, float boundaryChannelRatio, boolean hasCycle) {
+		StringBuilder s = new StringBuilder();
+		boolean accept = true;
+		if (Options.prognosticate) {
+			if (Options.bigToSmallBlobRatio > 0
+					&& bigToSmallBlobRatio > Options.bigToSmallBlobRatio) {
+				s.append("1,");
+				accept = false;
+			}
+			if (Options.loadRatio > 0 && loadRatio > Options.loadRatio) {
+				s.append("2,");
+				accept = false;
+			}
+			if (Options.blobToNodeRatio > 0
+					&& blobToNodeRatio > Options.blobToNodeRatio) {
+				s.append("3,");
+				accept = false;
+			}
+			if (Options.boundaryChannelRatio > 0
+					&& boundaryChannelRatio < Options.boundaryChannelRatio) {
+				s.append("4,");
+				accept = false;
+			}
+			if (hasCycle) {
+				s.append("5,");
+				accept = false;
+			}
+		}
+
+		try {
+			writer.write(String.format("%s\t\t",
+					accept ? "Accepted" : s.toString()));
+		} catch (IOException e) {
+
+		}
+		return accept;
 	}
 
 	/**
@@ -157,6 +197,8 @@ private static void writeHeader(FileWriter writer) {
 			writer.write("\t\t");
 			writer.write(String.format("%.7s", "hasCycles"));
 			writer.write("\t\t");
+			writer.write(String.format("%.7s", "A/R")); // Accepted or Rejected.
+			writer.write("\t\t");
 			writer.write(String.format("%.7s", "time"));
 			// writer.write("\t\t");
 			writer.flush();

From 4ce55b2fc246547edcba954f5d61404ae1c62d5c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 11:53:23 +0800
Subject: [PATCH 803/881] TimeLogProcessor:Very first timing is Init time.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 1919e916..857b1be1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -52,7 +52,7 @@ private static Map<String, Integer> processRunTime(String appName,
 		File outFile = new File(outDir, "processedRunTime.txt");
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
-		String cfgPrefix = "1";
+		String cfgPrefix = "Init";
 		int i = 0;
 		Map<String, Integer> ret = new HashMap<>(5000);
 		int min = Integer.MAX_VALUE;
@@ -109,7 +109,7 @@ private static Map<String, Integer> process(BufferedReader reader,
 			FileWriter writer, String startString, boolean isms, int timepos)
 			throws IOException {
 		String line;
-		String cfgPrefix = "1";
+		String cfgPrefix = "Init";
 		int i = 0;
 		int timeUnitLength = 1;
 		if (isms)

From 291a453814e03108c5d8e1a195ad48616d0141f8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 11:55:56 +0800
Subject: [PATCH 804/881] Shorten to fit to the tab size. Accepted->Acptd

---
 .../impl/distributed/runtimer/GraphPropertyPrognosticator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index c71dddf0..8aec328a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -93,7 +93,7 @@ private boolean decide(float bigToSmallBlobRatio, float loadRatio,
 
 		try {
 			writer.write(String.format("%s\t\t",
-					accept ? "Accepted" : s.toString()));
+					accept ? "Acptd" : s.toString()));
 		} catch (IOException e) {
 
 		}

From 848d5f0e6610ae8e7ff8ad7e8a7371495df81cfd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 12:05:49 +0800
Subject: [PATCH 805/881] Options.timeOut added.

---
 src/edu/mit/streamjit/impl/distributed/common/Options.java  | 5 ++++-
 .../streamjit/impl/distributed/runtimer/OnlineTuner.java    | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index e358d163..fe0494d5 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -172,6 +172,8 @@ public final class Options {
 
 	public static final int boundaryChannelRatio;
 
+	public static final boolean timeOut;
+
 	static {
 		Properties prop = loadProperties();
 		printOutputCountPeriod = Integer.parseInt(prop
@@ -205,7 +207,7 @@ public final class Options {
 		blobToNodeRatio = Integer.parseInt(prop.getProperty("blobToNodeRatio"));
 		boundaryChannelRatio = Integer.parseInt(prop
 				.getProperty("boundaryChannelRatio"));
-
+		timeOut = Boolean.parseBoolean(prop.getProperty("timeOut"));
 	}
 
 	public static Properties getProperties() {
@@ -231,6 +233,7 @@ public static Properties getProperties() {
 		setProperty(prop, "loadRatio", loadRatio);
 		setProperty(prop, "blobToNodeRatio", blobToNodeRatio);
 		setProperty(prop, "boundaryChannelRatio", boundaryChannelRatio);
+		setProperty(prop, "timeOut", timeOut);
 		return prop;
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 14c90524..2cac49f1 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -75,7 +75,11 @@ private void tune() {
 		int round = 0;
 		// Keeps track of the current best time. Uses this to discard bad cfgs
 		// early.
-		long currentBestTime = Long.MAX_VALUE;
+		long currentBestTime;
+		if (Options.timeOut)
+			currentBestTime = Long.MAX_VALUE;
+		else
+			currentBestTime = 0;
 		try {
 			startTuner();
 			Pair<Boolean, Long> ret;

From d2543d61d6c1095093b7069b4358f33cd02c145e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 13:27:11 +0800
Subject: [PATCH 806/881] Utils.fileWriter(dirName,fileName) added.

A convenient method.
---
 .../impl/distributed/TimeLoggers.java         |  8 ++---
 .../impl/distributed/Visualizer.java          |  3 +-
 .../impl/distributed/common/Utils.java        | 31 ++++++++++++++++---
 .../profiler/ProfileElementLoggers.java       |  3 +-
 .../runtimer/GraphPropertyPrognosticator.java |  3 +-
 5 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index b5fdd509..3571a418 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -30,11 +30,9 @@ public class TimeLoggers {
 	public static class FileTimeLogger extends TimeLoggerImpl {
 
 		public FileTimeLogger(String appName) {
-			super(Utils.fileWriter(String.format("%s%scompileTime.txt",
-					appName, File.separator)), Utils.fileWriter(String.format(
-					"%s%srunTime.txt", appName, File.separator)), Utils
-					.fileWriter(String.format("%s%sdrainTime.txt", appName,
-							File.separator)));
+			super(Utils.fileWriter(appName, "compileTime.txt"), Utils
+					.fileWriter(appName, "runTime.txt"), Utils.fileWriter(
+					appName, "drainTime.txt"));
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 26b257f0..9a94c1a2 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -109,8 +109,7 @@ private class DOTstreamVisitor extends StreamVisitor {
 			private final FileWriter writer;
 
 			DOTstreamVisitor() {
-				writer = Utils.fileWriter(String.format("%s%sstreamgraph.dot",
-						appName, File.separator));
+				writer = Utils.fileWriter(appName, "streamgraph.dot");
 			}
 
 			private void initilizeDot() {
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 17bb0adc..7a4f31ef 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -163,13 +163,37 @@ public static boolean rename(String appName, String fileName) {
 		return true;
 	}
 
+	/**
+	 * Returns a {@link FileWriter} of the file "dirName/fileName" with append =
+	 * false. Creates the file if it not exists. Suppresses {@link IOException}
+	 * and returns null if exception occurred. This method is added to keep
+	 * other classes clean.
+	 * 
+	 * @return {@link FileWriter} or null.
+	 */
+	public static FileWriter fileWriter(String dirName, String fileName) {
+		return fileWriter(dirName, fileName, false);
+	}
+
+	/**
+	 * Returns a {@link FileWriter} of the file "dirName/fileName". Creates the
+	 * file if it not exists. Suppresses {@link IOException} and returns null if
+	 * exception occurred. This method is added to keep other classes clean.
+	 * 
+	 * @return {@link FileWriter} or null.
+	 */
+	public static FileWriter fileWriter(String dirName, String fileName,
+			boolean append) {
+		String fullFileName = String.format("%s%s%s", dirName, File.separator,
+				fileName);
+		return fileWriter(fullFileName, append);
+	}
 	/**
 	 * Creates and returns a {@link FileWriter} with append = false. Suppresses
 	 * {@link IOException} and returns null if exception occurred. This method
 	 * is added to keep other classes clean.
 	 * 
-	 * @param name
-	 * @return
+	 * @return {@link FileWriter} or null.
 	 */
 	public static FileWriter fileWriter(String name) {
 		return fileWriter(name, false);
@@ -180,8 +204,7 @@ public static FileWriter fileWriter(String name) {
 	 * and returns null if exception occurred. This method is added to keep
 	 * other classes clean.
 	 * 
-	 * @param name
-	 * @return
+	 * @return {@link FileWriter} or null.
 	 */
 	public static FileWriter fileWriter(String name, boolean append) {
 		FileWriter fw = null;
diff --git a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
index 3d60c0cd..e99d2d3f 100755
--- a/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/profiler/ProfileElementLoggers.java
@@ -23,8 +23,7 @@ public static class FileProfileElementLogger
 				ProfileElementLoggerImpl {
 
 		public FileProfileElementLogger(String appName) {
-			super(Utils.fileWriter(String.format("%s%sprofile.txt", appName,
-					File.separator)));
+			super(Utils.fileWriter(appName, "profile.txt"));
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
index 8aec328a..0d54175a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
@@ -33,8 +33,7 @@ public class GraphPropertyPrognosticator implements ConfigurationPrognosticator
 
 	public GraphPropertyPrognosticator(StreamJitApp<?, ?> app) {
 		this.app = app;
-		this.writer = Utils.fileWriter(String.format("%s%sGraphProperty.txt",
-				app.name, File.separator));
+		this.writer = Utils.fileWriter(app.name, "GraphProperty.txt");
 		writeHeader(writer);
 		paths = app.paths();
 	}

From 36e922d706180b15c6b3b11166fd0c0d4c302ef7 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 13:54:28 +0800
Subject: [PATCH 807/881] TailChannels.OutputCountPrinter prints in a file.

---
 .../impl/distributed/TailChannels.java        | 37 ++++++++++++-------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 7e6e5954..d6f6ca59 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -3,6 +3,8 @@
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -16,6 +18,7 @@
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionInfo;
 import edu.mit.streamjit.impl.distributed.common.Connection.ConnectionProvider;
 import edu.mit.streamjit.impl.distributed.common.Options;
+import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.BlockingInputChannel;
 
 public class TailChannels {
@@ -105,6 +108,8 @@ public void stopLogging() {
 	 */
 	private static class OutputCountPrinter {
 
+		private final String appName;
+
 		private final TailChannel tailChannel;
 
 		/**
@@ -112,34 +117,40 @@ private static class OutputCountPrinter {
 		 */
 		private int lastCount;
 
+		private FileWriter writer;
+
+		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
+
 		private ScheduledExecutorService scheduledExecutorService;
 
-		OutputCountPrinter(TailChannel tailChannel) {
+		OutputCountPrinter(TailChannel tailChannel, String appName) {
 			this.tailChannel = tailChannel;
+			this.appName = appName;
 			printOutputCount();
 		}
 
 		private void printOutputCount() {
 			if (Options.printOutputCountPeriod < 1)
 				return;
-
+			writer = Utils.fileWriter(appName, "outputCount.txt", true);
 			lastCount = 0;
 			scheduledExecutorService = Executors
 					.newSingleThreadScheduledExecutor();
 			scheduledExecutorService.scheduleAtFixedRate(
 					new Runnable() {
-
 						@Override
 						public void run() {
 							int currentCount = tailChannel.count();
 							int newOutputs = currentCount - lastCount;
 							lastCount = currentCount;
-							System.out.println(String
-									.format("Outputs: since started - %d, during last %d ms - %d",
-											currentCount,
-											Options.printOutputCountPeriod,
-											newOutputs));
-
+							String msg = String.format("%d\t\t%d\t\t%d\n",
+									rb.getUptime(), currentCount, newOutputs);
+							try {
+								writer.write(msg);
+								writer.flush();
+							} catch (IOException e) {
+								e.printStackTrace();
+							}
 						}
 					}, Options.printOutputCountPeriod,
 					Options.printOutputCountPeriod,
@@ -199,7 +210,7 @@ public AbstractBlockingTailChannel(Buffer buffer,
 				pLogger.start();
 			}
 			if (Options.printOutputCountPeriod > 0)
-				outputCountPrinter = new OutputCountPrinter(this);
+				outputCountPrinter = new OutputCountPrinter(this, appName);
 		}
 
 		@Override
@@ -219,8 +230,7 @@ public int count() {
 		}
 
 		protected long normalizedTime(long time) {
-			return (Options.outputCount * time)
-					/ (totalCount - skipCount);
+			return (Options.outputCount * time) / (totalCount - skipCount);
 		}
 
 		/**
@@ -228,8 +238,7 @@ protected long normalizedTime(long time) {
 		 * <code>time=unnormalizedTime(normalizedTime(time))</code>
 		 */
 		protected long unnormalizedTime(long time) {
-			return (time * (totalCount - skipCount))
-					/ Options.outputCount;
+			return (time * (totalCount - skipCount)) / Options.outputCount;
 		}
 	}
 

From 08ae9ab837aa5bac9f9ccdd52b8aac01ecbcdce9 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 4 Mar 2015 15:53:01 +0800
Subject: [PATCH 808/881] Logs searchTime

The time that open tuner takes to deliver the next configuration.
---
 .../mit/streamjit/impl/common/TimeLogger.java |  7 +++++
 .../impl/distributed/TimeLoggers.java         | 26 ++++++++++++++-----
 .../distributed/runtimer/OnlineTuner.java     |  8 ++++++
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index 23d5479b..4cd35349 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -134,4 +134,11 @@ public interface TimeLogger {
 	 *            empty string if the prefix is unknown.
 	 */
 	public void newConfiguration(String cfgPrefix);
+
+	/**
+	 * Logs the time taken to get a new configuration from the OpenTuner.
+	 * 
+	 * @param time
+	 */
+	public void logSearchTime(long time);
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
index 3571a418..593a91e4 100644
--- a/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
+++ b/src/edu/mit/streamjit/impl/distributed/TimeLoggers.java
@@ -1,6 +1,5 @@
 package edu.mit.streamjit.impl.distributed;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
@@ -32,7 +31,8 @@ public static class FileTimeLogger extends TimeLoggerImpl {
 		public FileTimeLogger(String appName) {
 			super(Utils.fileWriter(appName, "compileTime.txt"), Utils
 					.fileWriter(appName, "runTime.txt"), Utils.fileWriter(
-					appName, "drainTime.txt"));
+					appName, "drainTime.txt"), Utils.fileWriter(appName,
+					"searchTime.txt"));
 		}
 	}
 
@@ -97,6 +97,10 @@ public void drainDataCollectionStarted() {
 		@Override
 		public void drainDataCollectionFinished(String msg) {
 		}
+
+		@Override
+		public void logSearchTime(long time) {
+		}
 	}
 
 	/**
@@ -106,7 +110,7 @@ public void drainDataCollectionFinished(String msg) {
 	public static class PrintTimeLogger extends TimeLoggerImpl {
 
 		public PrintTimeLogger() {
-			super(System.out, System.out, System.out);
+			super(System.out, System.out, System.out, System.out);
 		}
 	}
 
@@ -118,6 +122,8 @@ private static class TimeLoggerImpl implements TimeLogger {
 
 		private final OutputStreamWriter runTimeWriter;
 
+		private final OutputStreamWriter searchTimeWriter;
+
 		private int reconfigNo = 0;
 
 		private Stopwatch compileTimeSW = null;
@@ -129,16 +135,17 @@ private static class TimeLoggerImpl implements TimeLogger {
 		private Stopwatch tuningRoundSW = null;
 
 		TimeLoggerImpl(OutputStream compileOS, OutputStream runOs,
-				OutputStream drainOs) {
+				OutputStream drainOs, OutputStream searchOs) {
 			this(getOSWriter(compileOS), getOSWriter(runOs),
-					getOSWriter(drainOs));
+					getOSWriter(drainOs), getOSWriter(searchOs));
 		}
 
 		TimeLoggerImpl(OutputStreamWriter compileW, OutputStreamWriter runW,
-				OutputStreamWriter drainW) {
+				OutputStreamWriter drainW, OutputStreamWriter searchW) {
 			compileTimeWriter = compileW;
 			runTimeWriter = runW;
 			drainTimeWriter = drainW;
+			searchTimeWriter = searchW;
 		}
 
 		@Override
@@ -245,6 +252,7 @@ public void newConfiguration(String cfgPrefix) {
 							prevcfgPrefix);
 			write(compileTimeWriter, msg);
 			write(runTimeWriter, msg);
+			write(searchTimeWriter, msg);
 			write(drainTimeWriter, msg1);
 			prevcfgPrefix = cfgPrefix;
 		}
@@ -279,5 +287,11 @@ private void write(OutputStreamWriter osWriter, String msg) {
 				}
 			}
 		}
+
+		@Override
+		public void logSearchTime(long time) {
+			write(searchTimeWriter,
+					String.format("Search time is %dms\n", time));
+		}
 	}
 }
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2cac49f1..2f4e6251 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -10,6 +10,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -80,6 +83,7 @@ private void tune() {
 			currentBestTime = Long.MAX_VALUE;
 		else
 			currentBestTime = 0;
+		Stopwatch searchTimeSW = Stopwatch.createStarted();
 		try {
 			startTuner();
 			Pair<Boolean, Long> ret;
@@ -87,6 +91,8 @@ private void tune() {
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
 				String cfgJson = tuner.readLine();
+				logger.logSearchTime(searchTimeSW
+						.elapsed(TimeUnit.MILLISECONDS));
 				if (cfgJson == null) {
 					System.err.println("OpenTuner closed unexpectly.");
 					break;
@@ -107,6 +113,8 @@ private void tune() {
 							? time : currentBestTime;
 					prognosticator.time(ret.second);
 					tuner.writeLine(new Double(ret.second).toString());
+					searchTimeSW.reset();
+					searchTimeSW.start();
 				} else {
 					tuner.writeLine("exit");
 					break;

From 43033efcff604c326868c331e5dd252573d26cf5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Mar 2015 08:44:38 +0800
Subject: [PATCH 809/881] Removed downloading *jar in downloadscript

---
 jarapp.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index c57aca79..e15b8d11 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -145,7 +145,7 @@ mkdir -p $app
 cd $app
 mkdir -p $mainClass
 scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
-scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties,*.jar\} .
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties\} .
 	</echo>
 	</target>
 </project>

From 70f1fafbed05ba293e56f9ddbec4718e2fca8da3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Mar 2015 10:01:37 +0800
Subject: [PATCH 810/881] Ant: Uses rsync to download a full app dir.

---
 jarapp.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/jarapp.xml b/jarapp.xml
index e15b8d11..4686f7c8 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -146,6 +146,8 @@ cd $app
 mkdir -p $mainClass
 scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
 scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties\} .
+#to download everything.
+#rsync -avh --progress ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app .
 	</echo>
 	</target>
 </project>

From e5cd1ebe1478c4343053848d1eb18d1ddcdc4539 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Mar 2015 10:27:20 +0800
Subject: [PATCH 811/881] TimeLogProcessor.processEvaltxt() added.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 857b1be1..714cf7ca 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -332,4 +332,29 @@ else if (line.contains("False"))
 				"Total=%f, TrueCount=%d(%f), FalseCount=%d(%f)", total,
 				truecnt, (truecnt / total), falsecnt, (falsecnt / total)));
 	}
+
+	public static void processEvaltxt(String appName) throws IOException {
+		File summaryDir = new File(String.format("%s%ssummary", appName,
+				File.separator));
+		Utils.createDir(summaryDir.getPath());
+		BufferedReader reader = new BufferedReader(new FileReader(
+				String.format("%s%sevaluation.txt", appName, File.separator)));
+		File outFile = new File(summaryDir, "processedEvaluation.txt");
+		FileWriter writer = new FileWriter(outFile, false);
+		String line;
+		int i = 0;
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("Evaluation")) {
+				String[] arr = line.split(" ");
+				String time = arr[3].trim();
+				time = time.substring(0, time.length() - 2);
+				int val = Integer.parseInt(time);
+				String data = String.format("%-6d\t%-6d\n", ++i, val);
+				writer.write(data);
+			}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+	}
 }

From 199a6d7a67ae8985bbe3ec14f21f7a15cff1d24b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Mar 2015 11:17:37 +0800
Subject: [PATCH 812/881] TimeLogProcessor.processVerifycaionRun() added.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 714cf7ca..9c903500 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -12,6 +12,7 @@
 import java.util.List;
 import java.util.Map;
 
+import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
 /**
@@ -333,6 +334,11 @@ else if (line.contains("False"))
 				truecnt, (truecnt / total), falsecnt, (falsecnt / total)));
 	}
 
+	/**
+	 * If you evaluate a same configuration for several times by setting the
+	 * {@link Options#evaluationCount} value, then use this method to process
+	 * the output evaluation.txt.
+	 */
 	public static void processEvaltxt(String appName) throws IOException {
 		File summaryDir = new File(String.format("%s%ssummary", appName,
 				File.separator));
@@ -357,4 +363,50 @@ public static void processEvaltxt(String appName) throws IOException {
 		reader.close();
 		writer.close();
 	}
+
+	/**
+	 * If you run a set of configurations in a cyclic manner by setting
+	 * {@link Options#verificationCount}, then use this method to process the
+	 * runTime.txt.
+	 */
+	private static void processVerifycaionRunTime(String appName, File outDir,
+			String cfgName) throws IOException {
+		BufferedReader reader = new BufferedReader(new FileReader(
+				String.format("%s%srunTime.txt", appName, File.separator)));
+		File outFile = new File(outDir, String.format("processed%s.txt",
+				cfgName));
+		FileWriter writer = new FileWriter(outFile, false);
+		String line;
+		String cfgPrefix = "Init";
+		int i = 0;
+		int min = Integer.MAX_VALUE;
+		while ((line = reader.readLine()) != null) {
+			if (line.startsWith("----------------------------"))
+				cfgPrefix = cfgString(line);
+			if (cfgName.equals(cfgPrefix))
+				if (line.startsWith("Execution")) {
+					String[] arr = line.split(" ");
+					String time = arr[3].trim();
+					time = time.substring(0, time.length() - 2);
+					int val = Integer.parseInt(time);
+					if (val < 1)
+						val = 2 * min;
+					min = Math.min(min, val);
+					String data = String.format("%-6d\t%-6s\t%-6d\t%-6d\n",
+							++i, cfgPrefix, val, min);
+					writer.write(data);
+				}
+		}
+		writer.flush();
+		reader.close();
+		writer.close();
+	}
+
+	public static void processVerifycaionRun(String appName) throws IOException {
+		File summaryDir = new File(String.format("%s%ssummary", appName,
+				File.separator));
+		Utils.createDir(summaryDir.getPath());
+		processVerifycaionRunTime(appName, summaryDir,
+				"final_NestedSplitJoinCore.cfg");
+	}
 }

From 5ae7a2e466761ea93e3225819399cf8a1b396f62 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 5 Mar 2015 11:19:56 +0800
Subject: [PATCH 813/881] Output file name changed. Eval_appName->evaluation

---
 .../mit/streamjit/impl/distributed/runtimer/OnlineTuner.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 2f4e6251..b200ffaa 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -360,8 +360,9 @@ private void processRunningTimes(List<Long> runningTimes,
 		}
 
 		private FileWriter writer() throws IOException {
-			FileWriter writer = new FileWriter(String.format("%s%sEval_%s.txt",
-					app.name, File.separator, app.name), true);
+			FileWriter writer = new FileWriter(String.format(
+					"%s%sevaluation.txt", app.name, File.separator, app.name),
+					true);
 			writer.write("##########################################################");
 			Properties prop = Options.getProperties();
 			prop.store(writer, "");

From 7921ff9f7ba22b8d46ed6075ca99840ca6b3ea7b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 07:19:58 +0800
Subject: [PATCH 814/881] TailChannels log reportingTime event.

TailChannels log reportingTime event to OutputCountPrinter's file.
---
 .../impl/distributed/TailChannels.java        | 35 ++++++++++++++++---
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index d6f6ca59..894d65f4 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -161,6 +161,20 @@ private void stop() {
 			if (scheduledExecutorService != null)
 				scheduledExecutorService.shutdown();
 		}
+
+		/**
+		 * Logs the time reporting event.
+		 * 
+		 * TODO: This method is just for debugging purpose, Remove this method
+		 * and its usage later.
+		 */
+		private void reportingTime(long time) {
+			if (writer != null)
+				try {
+					writer.write(String.format("Reporting time=%d...\n", time));
+				} catch (Exception e) {
+				}
+		}
 	}
 
 	private static abstract class AbstractBlockingTailChannel
@@ -240,6 +254,17 @@ protected long normalizedTime(long time) {
 		protected long unnormalizedTime(long time) {
 			return (time * (totalCount - skipCount)) / Options.outputCount;
 		}
+
+		/**
+		 * Logs the time reporting event.
+		 * 
+		 * TODO: This method is just for debugging purpose, Remove this method
+		 * and its usage later.
+		 */
+		protected void reportingTime(long time) {
+			if (outputCountPrinter != null)
+				outputCountPrinter.reportingTime(time);
+		}
 	}
 
 	public static final class BlockingTailChannel1
@@ -316,6 +341,7 @@ public long getFixedOutputTime() throws InterruptedException {
 			steadyLatch.await();
 			stopwatch.stop();
 			long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+			reportingTime(time);
 			return normalizedTime(time);
 		}
 
@@ -334,11 +360,11 @@ public long getFixedOutputTime(long timeout)
 				Thread.sleep(100);
 			}
 
-			if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > timeout)
-				return -1;
-
 			stopwatch.stop();
 			long time = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+			reportingTime(time);
+			if (time > timeout)
+				return -1;
 			return normalizedTime(time);
 		}
 
@@ -432,6 +458,7 @@ public long getFixedOutputTime() throws InterruptedException {
 			while (stopWatch.isRunning())
 				Thread.sleep(250);
 			long time = stopWatch.elapsed(TimeUnit.MILLISECONDS);
+			reportingTime(time);
 			return normalizedTime(time);
 		}
 
@@ -451,7 +478,7 @@ public long getFixedOutputTime(long timeout)
 			}
 
 			long time = stopWatch.elapsed(TimeUnit.MILLISECONDS);
-
+			reportingTime(time);
 			if (time > timeout)
 				return -1;
 			else

From aa162a6a9c9b0809b88541ffd9d4ca2ef5584b75 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 07:42:24 +0800
Subject: [PATCH 815/881] tailChannel()'s method args rearranged.

---
 .../impl/distributed/StreamJitAppManager.java  | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 8fc54bab..75783d04 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -341,16 +341,18 @@ else if (headconInfo instanceof AsyncTCPConnectionInfo)
 					"No tail buffer in the passed bufferMap.");
 
 		int skipCount = Math.max(Options.outputCount, multiplier * 5);
-		tailChannel = tailChannel(bufferMap.get(tailToken),
-				controller.getConProvider(), tailconInfo, "tailChannel - "
-						+ tailToken.toString(), 0, skipCount,
-				Options.outputCount, app.name);
+		tailChannel = tailChannel(bufferMap.get(tailToken), tailconInfo,
+				skipCount);
 	}
 
-	private TailChannel tailChannel(Buffer buffer,
-			ConnectionProvider conProvider, ConnectionInfo conInfo,
-			String bufferTokenName, int debugLevel, int skipCount,
-			int steadyCount, String appName) {
+	private TailChannel tailChannel(Buffer buffer, ConnectionInfo conInfo,
+			int skipCount) {
+		String appName = app.name;
+		int steadyCount = Options.outputCount;
+		int debugLevel = 0;
+		String bufferTokenName = "tailChannel - " + tailToken.toString();
+		ConnectionProvider conProvider = controller.getConProvider();
+
 		switch (Options.tailChannel) {
 			case 1 :
 				return new TailChannels.BlockingTailChannel1(buffer,

From 7b6808eca968eca64047fbb5c1d79038447dc813 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 07:58:42 +0800
Subject: [PATCH 816/881] TailChannels logs cfgPrefix

This is added for debugging purpose.
---
 .../impl/distributed/StreamJitAppManager.java |  7 +++--
 .../impl/distributed/TailChannels.java        | 31 ++++++++++++-------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
index 75783d04..46df8e42 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitAppManager.java
@@ -352,16 +352,17 @@ private TailChannel tailChannel(Buffer buffer, ConnectionInfo conInfo,
 		int debugLevel = 0;
 		String bufferTokenName = "tailChannel - " + tailToken.toString();
 		ConnectionProvider conProvider = controller.getConProvider();
-
+		String cfgPrefix = ConfigurationUtils.getConfigPrefix(app
+				.getConfiguration());
 		switch (Options.tailChannel) {
 			case 1 :
 				return new TailChannels.BlockingTailChannel1(buffer,
 						conProvider, conInfo, bufferTokenName, debugLevel,
-						skipCount, steadyCount, appName);
+						skipCount, steadyCount, appName, cfgPrefix);
 			default :
 				return new TailChannels.BlockingTailChannel2(buffer,
 						conProvider, conInfo, bufferTokenName, debugLevel,
-						skipCount, steadyCount, appName);
+						skipCount, steadyCount, appName, cfgPrefix);
 		}
 	}
 
diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 894d65f4..579a837b 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -163,17 +163,20 @@ private void stop() {
 		}
 
 		/**
-		 * Logs the time reporting event.
-		 * 
+		 * This method writes to the file in a non thread safe way. But this is
+		 * enough to serve the purpose.
+		 * <P>
 		 * TODO: This method is just for debugging purpose, Remove this method
 		 * and its usage later.
 		 */
-		private void reportingTime(long time) {
+		private boolean write(String msg) {
 			if (writer != null)
 				try {
-					writer.write(String.format("Reporting time=%d...\n", time));
+					writer.write(msg);
+					return true;
 				} catch (Exception e) {
 				}
+			return false;
 		}
 	}
 
@@ -191,6 +194,8 @@ private static abstract class AbstractBlockingTailChannel
 
 		private OutputCountPrinter outputCountPrinter = null;
 
+		private final String cfgPrefix;
+
 		protected abstract void releaseAndInitilize();
 
 		/**
@@ -212,11 +217,12 @@ private static abstract class AbstractBlockingTailChannel
 		public AbstractBlockingTailChannel(Buffer buffer,
 				ConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel, int skipCount,
-				int steadyCount, String appName) {
+				int steadyCount, String appName, String cfgPrefix) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel);
 			this.skipCount = skipCount;
 			this.totalCount = steadyCount + skipCount;
 			count = 0;
+			this.cfgPrefix = cfgPrefix;
 			if (Options.tune == 0) {
 				// TODO: Leaks this object from the constructor. May cause
 				// subtle bugs. Re-factor it.
@@ -262,8 +268,11 @@ protected long unnormalizedTime(long time) {
 		 * and its usage later.
 		 */
 		protected void reportingTime(long time) {
-			if (outputCountPrinter != null)
-				outputCountPrinter.reportingTime(time);
+			if (outputCountPrinter != null) {
+				String msg = String.format("Reporting...%s.cfg,time=%d\n",
+						cfgPrefix, time);
+				outputCountPrinter.write(msg);
+			}
 		}
 	}
 
@@ -300,9 +309,9 @@ public static final class BlockingTailChannel1
 		public BlockingTailChannel1(Buffer buffer,
 				ConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel, int skipCount,
-				int steadyCount, String appName) {
+				int steadyCount, String appName, String cfgPrefix) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel,
-					skipCount, steadyCount, appName);
+					skipCount, steadyCount, appName, cfgPrefix);
 			steadyLatch = new CountDownLatch(1);
 			skipLatch = new CountDownLatch(1);
 			this.skipLatchUp = true;
@@ -419,9 +428,9 @@ public static final class BlockingTailChannel2
 		public BlockingTailChannel2(Buffer buffer,
 				ConnectionProvider conProvider, ConnectionInfo conInfo,
 				String bufferTokenName, int debugLevel, int skipCount,
-				int steadyCount, String appName) {
+				int steadyCount, String appName, String cfgPrefix) {
 			super(buffer, conProvider, conInfo, bufferTokenName, debugLevel,
-					skipCount, steadyCount, appName);
+					skipCount, steadyCount, appName, cfgPrefix);
 			stopWatch = Stopwatch.createUnstarted();
 			skipLatch = new CountDownLatch(1);
 			this.skipLatchUp = true;

From c7a4c054bede242ce737d8bdcba78bd49c1131d2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 09:25:15 +0800
Subject: [PATCH 817/881] Writes header to tatalStats.txt

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 9c903500..e2e5d7dd 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -192,7 +192,7 @@ public static void summarize(String appName) throws IOException {
 		FileWriter verify = new FileWriter(String.format("%s%sverify.txt",
 				appName, File.separator));
 		int min = Integer.MAX_VALUE;
-
+		writer.write("cfg\tTRTime\tcomp\trun\tdrain\tmin\n");
 		for (int i = 1; i <= tuningRoundTime.size(); i++) {
 			String key = new Integer(i).toString();
 			Integer time = runTime.get(key);

From 65f221573d9e043459ac8ade2e2148aaee6937d0 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 09:30:13 +0800
Subject: [PATCH 818/881] Stop writing null vals to tatalStats.txt.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index e2e5d7dd..0aefd58a 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -199,6 +199,7 @@ public static void summarize(String appName) throws IOException {
 
 			if (time == null) {
 				System.err.println("No running time for round " + key);
+				continue;
 			} else if (time < min) {
 				verify.write(String.format("%s=%d\n", key, time));
 				min = time;

From 4a655df0b70beae53fb1cc92539b83d339b24dc2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 20:24:35 +0800
Subject: [PATCH 819/881] OnlineTuner logs all important method's exe time.

This added for debugging purpose.
---
 .../runtimer/MethodTimeLogger.java            | 196 ++++++++++++++++++
 .../distributed/runtimer/OnlineTuner.java     |  28 ++-
 2 files changed, 221 insertions(+), 3 deletions(-)
 create mode 100644 src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
new file mode 100644
index 00000000..c13c623d
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -0,0 +1,196 @@
+package edu.mit.streamjit.impl.distributed.runtimer;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
+
+import edu.mit.streamjit.impl.distributed.common.Utils;
+
+/**
+ * Logs the opentuner's method call times for debugging purpose.
+ * 
+ * @author sumanan
+ * @since 6 Mar, 2015
+ */
+public interface MethodTimeLogger {
+
+	void bStartTuner();
+	void eStartTuner();
+
+	void bHandleTermination();
+	void eHandleTermination();
+
+	void bNewCfg();
+	void eNewCfg(int round);
+
+	void bReconfigure();
+	void eReconfigure();
+
+	void bTuningFinished();
+	void eTuningFinished();
+
+	void bTerminate();
+	void eTerminate();
+
+	void bIntermediateDraining();
+	void eIntermediateDraining();
+
+	void bManagerReconfigure();
+	void eManagerReconfigure();
+
+	void bGetFixedOutputTime();
+	void eGetFixedOutputTime();
+
+	public static class MethodTimeLoggerImpl implements MethodTimeLogger {
+
+		private final OutputStreamWriter osWriter;
+
+		private final Stopwatch startTuner;
+		private final Stopwatch handleTermination;
+		private final Stopwatch newCfg;
+		private final Stopwatch reconfigure;
+		private final Stopwatch tuningFinished;
+		private final Stopwatch terminate;
+		private final Stopwatch intermediateDraining;
+		private final Stopwatch managerReconfigure;
+		private final Stopwatch getFixedOutputTime;
+
+		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
+
+		public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
+			this.osWriter = osWriter;
+			this.startTuner = Stopwatch.createUnstarted();
+			this.handleTermination = Stopwatch.createUnstarted();
+			this.newCfg = Stopwatch.createUnstarted();
+			this.reconfigure = Stopwatch.createUnstarted();
+			this.tuningFinished = Stopwatch.createUnstarted();
+			this.terminate = Stopwatch.createUnstarted();
+			this.intermediateDraining = Stopwatch.createUnstarted();
+			this.managerReconfigure = Stopwatch.createUnstarted();
+			this.getFixedOutputTime = Stopwatch.createUnstarted();
+		}
+
+		@Override
+		public void bStartTuner() {
+			begin(startTuner);
+		}
+
+		@Override
+		public void eStartTuner() {
+			end(startTuner, "startTuner");
+		}
+
+		@Override
+		public void bHandleTermination() {
+			begin(handleTermination);
+		}
+
+		@Override
+		public void eHandleTermination() {
+			end(handleTermination, "handleTermination");
+		}
+
+		@Override
+		public void bNewCfg() {
+			begin(newCfg);
+		}
+
+		@Override
+		public void eNewCfg(int round) {
+			end(newCfg, "newCfg");
+		}
+
+		@Override
+		public void bReconfigure() {
+			begin(reconfigure);
+		}
+
+		@Override
+		public void eReconfigure() {
+			end(reconfigure, "reconfigure");
+		}
+
+		@Override
+		public void bTuningFinished() {
+			begin(tuningFinished);
+		}
+
+		@Override
+		public void eTuningFinished() {
+			end(tuningFinished, "tuningFinished");
+		}
+
+		@Override
+		public void bTerminate() {
+			begin(terminate);
+		}
+
+		@Override
+		public void eTerminate() {
+			end(terminate, "terminate");
+		}
+
+		@Override
+		public void bIntermediateDraining() {
+			begin(intermediateDraining);
+		}
+
+		@Override
+		public void eIntermediateDraining() {
+			end(intermediateDraining, "intermediateDraining");
+		}
+
+		@Override
+		public void bManagerReconfigure() {
+			begin(managerReconfigure);
+		}
+
+		@Override
+		public void eManagerReconfigure() {
+			end(managerReconfigure, "managerReconfigure");
+		}
+
+		@Override
+		public void bGetFixedOutputTime() {
+			begin(getFixedOutputTime);
+		}
+
+		@Override
+		public void eGetFixedOutputTime() {
+			end(getFixedOutputTime, "getFixedOutputTime");
+		}
+
+		private void begin(Stopwatch sw) {
+			sw.reset();
+			sw.start();
+		}
+
+		private void end(Stopwatch sw, String methodName) {
+			sw.stop();
+			long uptime = rb.getUptime();
+			long elapsedtime = sw.elapsed(TimeUnit.MILLISECONDS);
+			write(String.format("%s:uptime=%d,elapsedtime=%d\n", methodName,
+					uptime, elapsedtime));
+		}
+
+		private void write(String msg) {
+			if (osWriter != null)
+				try {
+					osWriter.write(msg);
+					osWriter.flush();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+		}
+	}
+
+	public static class FileMethodTimeLogger extends MethodTimeLoggerImpl {
+		public FileMethodTimeLogger(String appName) {
+			super(Utils.fileWriter(appName, "OnlineTuner.txt"));
+		}
+	}
+}
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b200ffaa..07e27c17 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -24,6 +24,7 @@
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
+import edu.mit.streamjit.impl.distributed.runtimer.MethodTimeLogger.FileMethodTimeLogger;
 import edu.mit.streamjit.tuner.OpenTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
 import edu.mit.streamjit.util.ConfigurationUtils;
@@ -47,6 +48,7 @@ public class OnlineTuner implements Runnable {
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
 	private final Verifier verifier;
+	private final MethodTimeLogger mLogger;
 
 	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
@@ -60,6 +62,7 @@ public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.logger = logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
 		this.verifier = new Verifier();
+		this.mLogger = new FileMethodTimeLogger(app.name);
 	}
 
 	@Override
@@ -85,7 +88,9 @@ private void tune() {
 			currentBestTime = 0;
 		Stopwatch searchTimeSW = Stopwatch.createStarted();
 		try {
+			mLogger.bStartTuner();
 			startTuner();
+			mLogger.eStartTuner();
 			Pair<Boolean, Long> ret;
 
 			System.out.println("New tune run.............");
@@ -101,12 +106,18 @@ private void tune() {
 				// At the end of the tuning, Opentuner will send "Completed"
 				// msg. This means no more tuning.
 				if (cfgJson.equals("Completed")) {
+					mLogger.bHandleTermination();
 					handleTermination();
+					mLogger.eHandleTermination();
 					break;
 				}
 
+				mLogger.bNewCfg();
 				Configuration config = newCfg(++round, cfgJson);
+				mLogger.eNewCfg(round);
+				mLogger.bReconfigure();
 				ret = reconfigure(config, 2 * currentBestTime);
+				mLogger.eReconfigure();
 				if (ret.first) {
 					long time = ret.second;
 					currentBestTime = (time > 1 && currentBestTime > time)
@@ -123,10 +134,13 @@ private void tune() {
 
 		} catch (IOException e) {
 			e.printStackTrace();
+			mLogger.bTerminate();
 			terminate();
+			mLogger.eTerminate();
 		}
-
+		mLogger.bTuningFinished();
 		tuningFinished();
+		mLogger.eTuningFinished();
 	}
 
 	private void startTuner() throws IOException {
@@ -180,16 +194,24 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 			return new Pair<Boolean, Long>(true, -3l);
 
 		try {
-			if (!intermediateDraining())
+			mLogger.bIntermediateDraining();
+			boolean intermediateDraining = intermediateDraining();
+			mLogger.eIntermediateDraining();
+			if (!intermediateDraining)
 				return new Pair<Boolean, Long>(false, -4l);
 
 			drainer.setBlobGraph(app.blobGraph);
 			int multiplier = getMultiplier(config);
-			if (manager.reconfigure(multiplier)) {
+			mLogger.bManagerReconfigure();
+			boolean reconfigure = manager.reconfigure(multiplier);
+			mLogger.eManagerReconfigure();
+			if (reconfigure) {
 				// TODO: need to check the manager's status before passing the
 				// time. Exceptions, final drain, etc may causes app to stop
 				// executing.
+				mLogger.bGetFixedOutputTime();
 				time = manager.getFixedOutputTime(timeout);
+				mLogger.eGetFixedOutputTime();
 				logger.logRunTime(time);
 			} else {
 				time = -5l;

From f9b734d911cf3c8def7e0ff9b59eb0cec8dc1293 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 20:47:12 +0800
Subject: [PATCH 820/881] MethodTimeLogger writes in a teble format.

---
 .../impl/distributed/runtimer/MethodTimeLogger.java       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index c13c623d..3bed3a00 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -72,6 +72,8 @@ public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
 			this.intermediateDraining = Stopwatch.createUnstarted();
 			this.managerReconfigure = Stopwatch.createUnstarted();
 			this.getFixedOutputTime = Stopwatch.createUnstarted();
+			write("Method\t\t\tUptime\t\telapsedtime\n");
+			write("----------------------------------------------------\n");
 		}
 
 		@Override
@@ -173,8 +175,10 @@ private void end(Stopwatch sw, String methodName) {
 			sw.stop();
 			long uptime = rb.getUptime();
 			long elapsedtime = sw.elapsed(TimeUnit.MILLISECONDS);
-			write(String.format("%s:uptime=%d,elapsedtime=%d\n", methodName,
-					uptime, elapsedtime));
+			// write(String.format("%s:uptime=%d,elapsedtime=%d\n", methodName,
+			// uptime, elapsedtime));
+			write(String.format("%-22s\t%-12d\t%d\n", methodName, uptime,
+					elapsedtime));
 		}
 
 		private void write(String msg) {

From edee86010b50caab0baeea0ac5e016b4e976e7d4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 21:06:37 +0800
Subject: [PATCH 821/881] OnlineTuner's method logger logs tuning round time

---
 .../distributed/runtimer/MethodTimeLogger.java | 18 +++++++++++++++++-
 .../impl/distributed/runtimer/OnlineTuner.java |  2 ++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index 3bed3a00..f1b3d9de 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -45,6 +45,9 @@ public interface MethodTimeLogger {
 	void bGetFixedOutputTime();
 	void eGetFixedOutputTime();
 
+	void bTuningRound();
+	void eTuningRound();
+
 	public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 
 		private final OutputStreamWriter osWriter;
@@ -58,6 +61,7 @@ public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 		private final Stopwatch intermediateDraining;
 		private final Stopwatch managerReconfigure;
 		private final Stopwatch getFixedOutputTime;
+		private final Stopwatch tuningRound;
 
 		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
 
@@ -72,8 +76,9 @@ public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
 			this.intermediateDraining = Stopwatch.createUnstarted();
 			this.managerReconfigure = Stopwatch.createUnstarted();
 			this.getFixedOutputTime = Stopwatch.createUnstarted();
+			this.tuningRound = Stopwatch.createUnstarted();
 			write("Method\t\t\tUptime\t\telapsedtime\n");
-			write("----------------------------------------------------\n");
+			write("====================================================\n");
 		}
 
 		@Override
@@ -166,6 +171,17 @@ public void eGetFixedOutputTime() {
 			end(getFixedOutputTime, "getFixedOutputTime");
 		}
 
+		@Override
+		public void bTuningRound() {
+			begin(tuningRound);
+		}
+
+		@Override
+		public void eTuningRound() {
+			end(tuningRound, "tuningRound");
+			write("--------------------------------------------------\n");
+		}
+
 		private void begin(Stopwatch sw) {
 			sw.reset();
 			sw.start();
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 07e27c17..b8c7cd8a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -95,6 +95,7 @@ private void tune() {
 
 			System.out.println("New tune run.............");
 			while (manager.getStatus() != AppStatus.STOPPED) {
+				mLogger.bTuningRound();
 				String cfgJson = tuner.readLine();
 				logger.logSearchTime(searchTimeSW
 						.elapsed(TimeUnit.MILLISECONDS));
@@ -130,6 +131,7 @@ private void tune() {
 					tuner.writeLine("exit");
 					break;
 				}
+				mLogger.eTuningRound();
 			}
 
 		} catch (IOException e) {

From 824bc7fb821639d155877466aa3df2d9df4820ac Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 21:22:16 +0800
Subject: [PATCH 822/881] BugFix: Closes files.

---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 579a837b..020a235a 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -160,6 +160,12 @@ public void run() {
 		private void stop() {
 			if (scheduledExecutorService != null)
 				scheduledExecutorService.shutdown();
+			if (writer != null)
+				try {
+					writer.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
 		}
 
 		/**

From ab35e10323507d3d397132e31a029ba4b97baac4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 21:33:15 +0800
Subject: [PATCH 823/881] file name starts with small letter. Online->online

---
 .../streamjit/impl/distributed/runtimer/MethodTimeLogger.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index f1b3d9de..fc683b9e 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -210,7 +210,7 @@ private void write(String msg) {
 
 	public static class FileMethodTimeLogger extends MethodTimeLoggerImpl {
 		public FileMethodTimeLogger(String appName) {
-			super(Utils.fileWriter(appName, "OnlineTuner.txt"));
+			super(Utils.fileWriter(appName, "onlineTuner.txt"));
 		}
 	}
 }

From 85b7282eb478276eb8333b8510bff16e4ee76cea Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 23:08:26 +0800
Subject: [PATCH 824/881] MethodTimeLogger.newCfg() log cfgPrefix

---
 .../streamjit/impl/distributed/runtimer/MethodTimeLogger.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index fc683b9e..6d91981f 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -108,7 +108,7 @@ public void bNewCfg() {
 
 		@Override
 		public void eNewCfg(int round) {
-			end(newCfg, "newCfg");
+			end(newCfg, String.format("newCfg-%d", round));
 		}
 
 		@Override

From d59582c4407de60d4245dd2f211e979c1d5547af Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 23:19:53 +0800
Subject: [PATCH 825/881] DotVisualizer generates blobgraphs in svg format

---
 src/edu/mit/streamjit/impl/distributed/Visualizer.java | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index 9a94c1a2..a52d0081 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -211,13 +211,14 @@ public void newConfiguration(Configuration cfg) {
 		}
 
 		private void runDot(String file) {
+			String outFileFormat = "svg";
 			String fileName = String.format("./%s%s%s.dot", appName,
 					File.separator, file);
 			String outFileName = String.format(
-					"./%s%sconfigurations%s%s_%s.png", appName, File.separator,
-					File.separator, namePrefix, file);
-			ProcessBuilder pb = new ProcessBuilder("dot", "-Tpng", fileName,
-					"-o", outFileName);
+					"./%s%sconfigurations%s%s_%s.%s", appName, File.separator,
+					File.separator, namePrefix, file, outFileFormat);
+			ProcessBuilder pb = new ProcessBuilder("dot", "-T" + outFileFormat,
+					fileName, "-o", outFileName);
 			try {
 				Process p = pb.start();
 				// TODO: [20-2-2015]. I am commenting the following line for

From 5d264122fd089a4517b53c76d5bcfd2b08f84a20 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 23:29:57 +0800
Subject: [PATCH 826/881] PrintMethodTimeLogger added.

---
 .../runtimer/MethodTimeLogger.java            | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index 6d91981f..deca2fac 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -1,6 +1,7 @@
 package edu.mit.streamjit.impl.distributed.runtimer;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
@@ -65,6 +66,10 @@ public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 
 		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
 
+		public MethodTimeLoggerImpl(OutputStream os) {
+			this(getOSWriter(os));
+		}
+
 		public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
 			this.osWriter = osWriter;
 			this.startTuner = Stopwatch.createUnstarted();
@@ -191,8 +196,6 @@ private void end(Stopwatch sw, String methodName) {
 			sw.stop();
 			long uptime = rb.getUptime();
 			long elapsedtime = sw.elapsed(TimeUnit.MILLISECONDS);
-			// write(String.format("%s:uptime=%d,elapsedtime=%d\n", methodName,
-			// uptime, elapsedtime));
 			write(String.format("%-22s\t%-12d\t%d\n", methodName, uptime,
 					elapsedtime));
 		}
@@ -206,6 +209,12 @@ private void write(String msg) {
 					e.printStackTrace();
 				}
 		}
+
+		private static OutputStreamWriter getOSWriter(OutputStream os) {
+			if (os == null)
+				return null;
+			return new OutputStreamWriter(os);
+		}
 	}
 
 	public static class FileMethodTimeLogger extends MethodTimeLoggerImpl {
@@ -213,4 +222,10 @@ public FileMethodTimeLogger(String appName) {
 			super(Utils.fileWriter(appName, "onlineTuner.txt"));
 		}
 	}
+
+	public static class PrintMethodTimeLogger extends MethodTimeLoggerImpl {
+		public PrintMethodTimeLogger() {
+			super(System.out);
+		}
+	}
 }

From 925c04bcbb20f7e5e2e6f301d5ce8584413b14de Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 6 Mar 2015 23:35:16 +0800
Subject: [PATCH 827/881] NoMethodTimeLogger added.

---
 .../runtimer/MethodTimeLogger.java            | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index deca2fac..ceba3c39 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -49,6 +49,92 @@ public interface MethodTimeLogger {
 	void bTuningRound();
 	void eTuningRound();
 
+	/**
+	 * Logs nothing.
+	 */
+	public static class NoMethodTimeLogger implements MethodTimeLogger {
+
+		@Override
+		public void bStartTuner() {
+		}
+
+		@Override
+		public void eStartTuner() {
+		}
+
+		@Override
+		public void bHandleTermination() {
+		}
+
+		@Override
+		public void eHandleTermination() {
+		}
+
+		@Override
+		public void bNewCfg() {
+		}
+
+		@Override
+		public void eNewCfg(int round) {
+		}
+
+		@Override
+		public void bReconfigure() {
+		}
+
+		@Override
+		public void eReconfigure() {
+		}
+
+		@Override
+		public void bTuningFinished() {
+		}
+
+		@Override
+		public void eTuningFinished() {
+		}
+
+		@Override
+		public void bTerminate() {
+		}
+
+		@Override
+		public void eTerminate() {
+		}
+
+		@Override
+		public void bIntermediateDraining() {
+		}
+
+		@Override
+		public void eIntermediateDraining() {
+		}
+
+		@Override
+		public void bManagerReconfigure() {
+		}
+
+		@Override
+		public void eManagerReconfigure() {
+		}
+
+		@Override
+		public void bGetFixedOutputTime() {
+		}
+
+		@Override
+		public void eGetFixedOutputTime() {
+		}
+
+		@Override
+		public void bTuningRound() {
+		}
+
+		@Override
+		public void eTuningRound() {
+		}
+	}
+
 	public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 
 		private final OutputStreamWriter osWriter;
@@ -217,12 +303,18 @@ private static OutputStreamWriter getOSWriter(OutputStream os) {
 		}
 	}
 
+	/**
+	 * Writes the method call time info to appName/onlineTuner.txt file.
+	 */
 	public static class FileMethodTimeLogger extends MethodTimeLoggerImpl {
 		public FileMethodTimeLogger(String appName) {
 			super(Utils.fileWriter(appName, "onlineTuner.txt"));
 		}
 	}
 
+	/**
+	 * Prints the method call time info to the standard out.
+	 */
 	public static class PrintMethodTimeLogger extends MethodTimeLoggerImpl {
 		public PrintMethodTimeLogger() {
 			super(System.out);

From 9901f88b42f82f5363670e336acc2b542ae6760b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Mar 2015 07:59:20 +0800
Subject: [PATCH 828/881] MethodTimeLogger measures cfgManager's newConfg()

---
 .../runtimer/MethodTimeLogger.java            | 23 +++++++++++++++++++
 .../distributed/runtimer/OnlineTuner.java     |  5 +++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index ceba3c39..1f705236 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -49,6 +49,9 @@ public interface MethodTimeLogger {
 	void bTuningRound();
 	void eTuningRound();
 
+	void bCfgManagerNewcfg();
+	void eCfgManagerNewcfg();
+
 	/**
 	 * Logs nothing.
 	 */
@@ -133,6 +136,14 @@ public void bTuningRound() {
 		@Override
 		public void eTuningRound() {
 		}
+
+		@Override
+		public void bCfgManagerNewcfg() {
+		}
+
+		@Override
+		public void eCfgManagerNewcfg() {
+		}
 	}
 
 	public static class MethodTimeLoggerImpl implements MethodTimeLogger {
@@ -149,6 +160,7 @@ public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 		private final Stopwatch managerReconfigure;
 		private final Stopwatch getFixedOutputTime;
 		private final Stopwatch tuningRound;
+		private final Stopwatch CfgManagerNewcfg;
 
 		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
 
@@ -168,6 +180,7 @@ public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
 			this.managerReconfigure = Stopwatch.createUnstarted();
 			this.getFixedOutputTime = Stopwatch.createUnstarted();
 			this.tuningRound = Stopwatch.createUnstarted();
+			this.CfgManagerNewcfg = Stopwatch.createUnstarted();
 			write("Method\t\t\tUptime\t\telapsedtime\n");
 			write("====================================================\n");
 		}
@@ -273,6 +286,16 @@ public void eTuningRound() {
 			write("--------------------------------------------------\n");
 		}
 
+		@Override
+		public void bCfgManagerNewcfg() {
+			begin(CfgManagerNewcfg);
+		}
+
+		@Override
+		public void eCfgManagerNewcfg() {
+			end(CfgManagerNewcfg, "CfgManagerNewcfg");
+		}
+
 		private void begin(Stopwatch sw) {
 			sw.reset();
 			sw.start();
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index b8c7cd8a..8d16e0a4 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -189,7 +189,10 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		if (manager.getStatus() == AppStatus.STOPPED)
 			return new Pair<Boolean, Long>(false, 0l);
 
-		if (!cfgManager.newConfiguration(config))
+		mLogger.bCfgManagerNewcfg();
+		boolean validCfg = cfgManager.newConfiguration(config);
+		mLogger.eCfgManagerNewcfg();
+		if (!validCfg)
 			return new Pair<Boolean, Long>(true, -2l);
 
 		if (!prognosticator.prognosticate(config))

From 83f3c4e44495c71112842d9015a4b7d2642c8506 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Mar 2015 08:08:41 +0800
Subject: [PATCH 829/881] MethodTimeLogger logs prognosticate()'s call time.

---
 .../runtimer/MethodTimeLogger.java            | 31 ++++++++++++++++---
 .../distributed/runtimer/OnlineTuner.java     |  5 ++-
 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
index 1f705236..652bdeee 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
@@ -52,6 +52,9 @@ public interface MethodTimeLogger {
 	void bCfgManagerNewcfg();
 	void eCfgManagerNewcfg();
 
+	void bPrognosticate();
+	void ePrognosticate();
+
 	/**
 	 * Logs nothing.
 	 */
@@ -144,6 +147,14 @@ public void bCfgManagerNewcfg() {
 		@Override
 		public void eCfgManagerNewcfg() {
 		}
+
+		@Override
+		public void bPrognosticate() {
+		}
+
+		@Override
+		public void ePrognosticate() {
+		}
 	}
 
 	public static class MethodTimeLoggerImpl implements MethodTimeLogger {
@@ -160,7 +171,8 @@ public static class MethodTimeLoggerImpl implements MethodTimeLogger {
 		private final Stopwatch managerReconfigure;
 		private final Stopwatch getFixedOutputTime;
 		private final Stopwatch tuningRound;
-		private final Stopwatch CfgManagerNewcfg;
+		private final Stopwatch cfgManagerNewcfg;
+		private final Stopwatch prognosticate;
 
 		private RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
 
@@ -180,7 +192,8 @@ public MethodTimeLoggerImpl(OutputStreamWriter osWriter) {
 			this.managerReconfigure = Stopwatch.createUnstarted();
 			this.getFixedOutputTime = Stopwatch.createUnstarted();
 			this.tuningRound = Stopwatch.createUnstarted();
-			this.CfgManagerNewcfg = Stopwatch.createUnstarted();
+			this.cfgManagerNewcfg = Stopwatch.createUnstarted();
+			this.prognosticate = Stopwatch.createUnstarted();
 			write("Method\t\t\tUptime\t\telapsedtime\n");
 			write("====================================================\n");
 		}
@@ -288,12 +301,22 @@ public void eTuningRound() {
 
 		@Override
 		public void bCfgManagerNewcfg() {
-			begin(CfgManagerNewcfg);
+			begin(cfgManagerNewcfg);
 		}
 
 		@Override
 		public void eCfgManagerNewcfg() {
-			end(CfgManagerNewcfg, "CfgManagerNewcfg");
+			end(cfgManagerNewcfg, "CfgManagerNewcfg");
+		}
+
+		@Override
+		public void bPrognosticate() {
+			begin(prognosticate);
+		}
+
+		@Override
+		public void ePrognosticate() {
+			end(prognosticate, "prognosticate");
 		}
 
 		private void begin(Stopwatch sw) {
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
index 8d16e0a4..bbd0182a 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
@@ -195,7 +195,10 @@ private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
 		if (!validCfg)
 			return new Pair<Boolean, Long>(true, -2l);
 
-		if (!prognosticator.prognosticate(config))
+		mLogger.bPrognosticate();
+		boolean prog = prognosticator.prognosticate(config);
+		mLogger.ePrognosticate();
+		if (!prog)
 			return new Pair<Boolean, Long>(true, -3l);
 
 		try {

From 8942b7299c268bea2cdbe13a0456d347fd941bcc Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 25 Apr 2014 10:53:27 +0800
Subject: [PATCH 830/881] Comment modified

---
 .../impl/concurrent/ConcurrentStreamCompiler.java     | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
index 031e595d..572be91e 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
@@ -35,6 +35,7 @@
 import edu.mit.streamjit.impl.common.VerifyStreamGraph;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.common.drainer.BlobGraph;
+import edu.mit.streamjit.impl.distributed.DistributedStreamCompiler;
 import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
@@ -42,7 +43,14 @@
 
 /**
  * A stream compiler that partitions a streamgraph into multiple blobs and
- * execute it on multiple threads.
+ * execute them on a single node. This {@link StreamCompiler} can be used for
+ * following purposes
+ * <ol>
+ * <li>Single blob online tuning.
+ * <li>Multiple blobs on a single node. This will simulate
+ * {@link DistributedStreamCompiler} on a single node to find out deadlocks and
+ * other issues.
+ * </ol>
  * 
  * @author Sumanan sumanan@mit.edu
  * @since Apr 8, 2013
@@ -63,7 +71,6 @@ public ConcurrentStreamCompiler(int noOfBlobs) {
 	}
 
 	public ConcurrentStreamCompiler(Configuration cfg) {
-
 		IntParameter threadCount = cfg.getParameter("threadCount",
 				IntParameter.class);
 		this.noOfBlobs = threadCount.getValue();

From d8a4c60e19ab618c2eb727d6cc6b7dbb1bd79330 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 05:52:17 +0800
Subject: [PATCH 831/881] Gets noOfBlobs parameter from configuration

Configuration can specify the number of blobs that should be created.
If no noOfBlobs parameter exists, it will be set to 1 by default.
---
 .../concurrent/ConcurrentStreamCompiler.java     | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
index 572be91e..465a77f9 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
@@ -71,13 +71,15 @@ public ConcurrentStreamCompiler(int noOfBlobs) {
 	}
 
 	public ConcurrentStreamCompiler(Configuration cfg) {
-		IntParameter threadCount = cfg.getParameter("threadCount",
-				IntParameter.class);
-		this.noOfBlobs = threadCount.getValue();
-		if (noOfBlobs < 1)
-			throw new IllegalArgumentException(
-					"noOfBlobs should be 1 or greater");
-		this.noOfBlobs = noOfBlobs;
+		IntParameter nBlobs = cfg.getParameter("noOfBlobs", IntParameter.class);
+		if (nBlobs == null)
+			this.noOfBlobs = 1;
+		else {
+			this.noOfBlobs = nBlobs.getValue();
+			if (noOfBlobs < 1)
+				throw new IllegalArgumentException(
+						"noOfBlobs should be 1 or greater");
+		}
 	}
 
 	@Override

From a7405e5de1f38bc555b46cd9bf4724a4379fe47d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 05:58:25 +0800
Subject: [PATCH 832/881] Another constructor has been added.

---
 .../impl/concurrent/ConcurrentStreamCompiler.java        | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
index 465a77f9..7da5af36 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentStreamCompiler.java
@@ -59,9 +59,8 @@ public class ConcurrentStreamCompiler implements StreamCompiler {
 	int noOfBlobs;
 
 	/**
-	 * @param Patrions
-	 *            a stream graph up to noOfBlobs many blobs and executes each
-	 *            blob on each thread.
+	 * @param noOfBlobs
+	 *            Maximum number of blobs that can be created.
 	 */
 	public ConcurrentStreamCompiler(int noOfBlobs) {
 		if (noOfBlobs < 1)
@@ -70,6 +69,10 @@ public ConcurrentStreamCompiler(int noOfBlobs) {
 		this.noOfBlobs = noOfBlobs;
 	}
 
+	public ConcurrentStreamCompiler() {
+		this(1);
+	}
+
 	public ConcurrentStreamCompiler(Configuration cfg) {
 		IntParameter nBlobs = cfg.getParameter("noOfBlobs", IntParameter.class);
 		if (nBlobs == null)

From 1dbe071a8222c478f682240918b1080b94e05b63 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 06:36:34 +0800
Subject: [PATCH 833/881] Comment modified

---
 .../mit/streamjit/impl/common/VerifyStreamGraph.java | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/VerifyStreamGraph.java b/src/edu/mit/streamjit/impl/common/VerifyStreamGraph.java
index 0d04aa9a..d467475b 100644
--- a/src/edu/mit/streamjit/impl/common/VerifyStreamGraph.java
+++ b/src/edu/mit/streamjit/impl/common/VerifyStreamGraph.java
@@ -21,10 +21,14 @@
 
 /**
  * {@link VerifyStreamGraph} currently verifies a stream graph for following
- * correctness. 1) A filter instance should be added only once in the graph. 2)
- * {@link WeightedRoundrobinSplitter} has matching numbers of branches and
- * weights. 3) {@link WeightedRoundrobinJoiner} has matching numbers of weights
- * array and the input branches.
+ * correctness.
+ * <ol>
+ * <li>1) A filter instance should be added only once in the graph.
+ * <li>2) {@link WeightedRoundrobinSplitter} has matching numbers of branches
+ * and weights.
+ * <li>3) {@link WeightedRoundrobinJoiner} has matching numbers of weights array
+ * and the input branches.
+ * </ol>
  * 
  * @author Sumanan sumanan@mit.edu
  * @since May 9, 2013

From f0c40e4883615e88dbad9f2cfd6291ec163f899a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 07:37:25 +0800
Subject: [PATCH 834/881] WorkerMachine has been made to public

from package public.
---
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index 6018a908..a182f1bd 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -37,7 +37,7 @@ public final class WorkerMachine extends AbstractPartitionManager {
 
 	private final Set<Worker<?, ?>> workerset;
 
-	WorkerMachine(StreamJitApp app) {
+	public WorkerMachine(StreamJitApp app) {
 		super(app);
 		this.workerset = Workers.getAllWorkersInGraph(app.source);
 	}

From 012b2bd5a0beade527e1900550784c2d47a057fb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 07:38:56 +0800
Subject: [PATCH 835/881] ConcurrentBlobFactory has been added

---
 .../concurrent/ConcurrentBlobFactory.java     | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
new file mode 100644
index 00000000..762c0cef
--- /dev/null
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
@@ -0,0 +1,59 @@
+package edu.mit.streamjit.impl.concurrent;
+
+import java.util.Set;
+
+import edu.mit.streamjit.api.Worker;
+import edu.mit.streamjit.impl.blob.Blob;
+import edu.mit.streamjit.impl.blob.BlobFactory;
+import edu.mit.streamjit.impl.blob.DrainData;
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.Parameter;
+import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.WorkerMachine;
+import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+
+public class ConcurrentBlobFactory implements BlobFactory {
+
+	private int noOfBlobs;
+
+	private final ConfigurationManager cfgManager;
+
+	public ConcurrentBlobFactory(ConfigurationManager cfgManager, int noOfBlobs) {
+		this.cfgManager = cfgManager;
+		this.noOfBlobs = noOfBlobs;
+	}
+
+	/**
+	 * If {@link ConfigurationManager} is not passed as a constructor argument
+	 * then {@link WorkerMachine} will be used as default one.
+	 * 
+	 * @param noOfMachines
+	 */
+	public ConcurrentBlobFactory(int noOfBlobs) {
+		this(new WorkerMachine(null), noOfBlobs);
+	}
+
+	@Override
+	public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
+			int maxNumCores, DrainData initialState) {
+		return new Compiler2BlobFactory().makeBlob(workers, config,
+				maxNumCores, initialState);
+	}
+
+	@Override
+	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
+
+		Configuration concurrentCfg = cfgManager.getDefaultConfiguration(workers,
+				noOfBlobs);
+		if (!GlobalConstants.useCompilerBlob)
+			return concurrentCfg;
+
+		Configuration.Builder builder = Configuration.builder(concurrentCfg);
+		BlobFactory compilerBf = new Compiler2BlobFactory();
+		Configuration compilercfg = compilerBf.getDefaultConfiguration(workers);
+		for (Parameter p : compilercfg.getParametersMap().values())
+			builder.addParameter(p);
+		return builder.build();
+	}
+}

From 1da5f73128af2e231bd494dc402854104d2d1d80 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 07:49:26 +0800
Subject: [PATCH 836/881] Checks argument's preconditions.

---
 src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java | 4 ++++
 src/edu/mit/streamjit/impl/distributed/WorkerMachine.java | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
index 022a0efa..0ef6d822 100644
--- a/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
+++ b/src/edu/mit/streamjit/impl/distributed/HotSpotTuning.java
@@ -1,5 +1,7 @@
 package edu.mit.streamjit.impl.distributed;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -34,6 +36,8 @@ public HotSpotTuning(StreamJitApp app) {
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 			int noOfMachines) {
+		checkArgument(noOfMachines > 0, String.format(
+				"noOfMachines = %d, It must be > 0", noOfMachines));
 		PickHotSpots visitor = new PickHotSpots(noOfMachines);
 		app.streamGraph.visit(visitor);
 		return visitor.builder.build();
diff --git a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
index a182f1bd..300a1860 100644
--- a/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
+++ b/src/edu/mit/streamjit/impl/distributed/WorkerMachine.java
@@ -7,6 +7,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static com.google.common.base.Preconditions.*;
+
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Configuration.IntParameter;
@@ -45,6 +47,8 @@ public WorkerMachine(StreamJitApp app) {
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers,
 			int noOfMachines) {
+		checkArgument(noOfMachines > 0, String.format(
+				"noOfMachines = %d, It must be > 0", noOfMachines));
 		Configuration.Builder builder = Configuration.builder();
 		List<Integer> machinelist = new ArrayList<>(noOfMachines);
 		for (int i = 1; i <= noOfMachines; i++)

From 72ae579e82ee3c88e26d97d6a947795b8a22bb01 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 07:56:19 +0800
Subject: [PATCH 837/881] Global variable made final

---
 .../mit/streamjit/impl/distributed/DistributedBlobFactory.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index 134c727c..f487dfd9 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -33,7 +33,7 @@
  */
 public class DistributedBlobFactory implements BlobFactory {
 
-	private int noOfMachines;
+	private final int noOfMachines;
 
 	private final PartitionManager partitionManager;
 

From 324167839fd972bcc9b27ccf25b5ef586c4b640d Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 08:35:53 +0800
Subject: [PATCH 838/881] hashCode() and equals() method has been added

to ConcurrentBlobFactory
---
 .../concurrent/ConcurrentBlobFactory.java     | 34 +++++++++++++++++--
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
index 762c0cef..5b2a5c65 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
@@ -43,9 +43,8 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
-
-		Configuration concurrentCfg = cfgManager.getDefaultConfiguration(workers,
-				noOfBlobs);
+		Configuration concurrentCfg = cfgManager.getDefaultConfiguration(
+				workers, noOfBlobs);
 		if (!GlobalConstants.useCompilerBlob)
 			return concurrentCfg;
 
@@ -56,4 +55,33 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 			builder.addParameter(p);
 		return builder.build();
 	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result
+				+ ((cfgManager == null) ? 0 : cfgManager.hashCode());
+		result = prime * result + noOfBlobs;
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		ConcurrentBlobFactory other = (ConcurrentBlobFactory) obj;
+		if (cfgManager == null) {
+			if (other.cfgManager != null)
+				return false;
+		} else if (!cfgManager.equals(other.cfgManager))
+			return false;
+		if (noOfBlobs != other.noOfBlobs)
+			return false;
+		return true;
+	}
 }

From 77b533ee5a9f4432cc8321c07a0ffb09c9d4707a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 28 Apr 2014 08:40:56 +0800
Subject: [PATCH 839/881] Uses empty configuration if noOfMachines==1

There is nothing to tune if noOfMachines==1.
---
 .../streamjit/impl/concurrent/ConcurrentBlobFactory.java | 9 +++++++--
 .../impl/distributed/DistributedBlobFactory.java         | 7 +++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
index 5b2a5c65..3d7e3b49 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
@@ -43,8 +43,13 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
-		Configuration concurrentCfg = cfgManager.getDefaultConfiguration(
-				workers, noOfBlobs);
+		Configuration concurrentCfg;
+		if (this.noOfBlobs > 1)
+			concurrentCfg = cfgManager.getDefaultConfiguration(workers,
+					noOfBlobs);
+		else
+			concurrentCfg = Configuration.builder().build();
+
 		if (!GlobalConstants.useCompilerBlob)
 			return concurrentCfg;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index f487dfd9..c105ee4f 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -65,9 +65,12 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
+		Configuration distCfg;
+		if (this.noOfMachines > 1)
+			distCfg = partitionManager.getDefaultConfiguration(workers, noOfMachines);
+		else
+			distCfg = Configuration.builder().build();
 
-		Configuration distCfg = partitionManager.getDefaultConfiguration(
-				workers, noOfMachines);
 		if (!Options.useCompilerBlob)
 			return distCfg;
 

From 4ec23eb718ea1f9d2f980a82b9027a51ff092277 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Mar 2015 13:38:28 +0800
Subject: [PATCH 840/881] ConcurrentStrmCmplr changes after the cherrypick

Some naming issues resolved.
---
 .../concurrent/ConcurrentBlobFactory.java     | 25 +++++++++++--------
 .../impl/concurrent/ConcurrentDrainer.java    |  8 +++---
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
index 3d7e3b49..b9f1fc50 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentBlobFactory.java
@@ -10,17 +10,19 @@
 import edu.mit.streamjit.impl.common.Configuration.Parameter;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobFactory;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.PartitionManager;
 import edu.mit.streamjit.impl.distributed.WorkerMachine;
-import edu.mit.streamjit.impl.distributed.common.GlobalConstants;
+import edu.mit.streamjit.impl.distributed.common.Options;
 
 public class ConcurrentBlobFactory implements BlobFactory {
 
 	private int noOfBlobs;
 
-	private final ConfigurationManager cfgManager;
+	private final PartitionManager partitionManager;
 
-	public ConcurrentBlobFactory(ConfigurationManager cfgManager, int noOfBlobs) {
-		this.cfgManager = cfgManager;
+	public ConcurrentBlobFactory(PartitionManager partitionManager,
+			int noOfBlobs) {
+		this.partitionManager = partitionManager;
 		this.noOfBlobs = noOfBlobs;
 	}
 
@@ -45,12 +47,12 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		Configuration concurrentCfg;
 		if (this.noOfBlobs > 1)
-			concurrentCfg = cfgManager.getDefaultConfiguration(workers,
+			concurrentCfg = partitionManager.getDefaultConfiguration(workers,
 					noOfBlobs);
 		else
 			concurrentCfg = Configuration.builder().build();
 
-		if (!GlobalConstants.useCompilerBlob)
+		if (!Options.useCompilerBlob)
 			return concurrentCfg;
 
 		Configuration.Builder builder = Configuration.builder(concurrentCfg);
@@ -65,8 +67,9 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 	public int hashCode() {
 		final int prime = 31;
 		int result = 1;
-		result = prime * result
-				+ ((cfgManager == null) ? 0 : cfgManager.hashCode());
+		result = prime
+				* result
+				+ ((partitionManager == null) ? 0 : partitionManager.hashCode());
 		result = prime * result + noOfBlobs;
 		return result;
 	}
@@ -80,10 +83,10 @@ public boolean equals(Object obj) {
 		if (getClass() != obj.getClass())
 			return false;
 		ConcurrentBlobFactory other = (ConcurrentBlobFactory) obj;
-		if (cfgManager == null) {
-			if (other.cfgManager != null)
+		if (partitionManager == null) {
+			if (other.partitionManager != null)
 				return false;
-		} else if (!cfgManager.equals(other.cfgManager))
+		} else if (!partitionManager.equals(other.partitionManager))
 			return false;
 		if (noOfBlobs != other.noOfBlobs)
 			return false;
diff --git a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
index 75e49d03..0e4b73d0 100644
--- a/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
+++ b/src/edu/mit/streamjit/impl/concurrent/ConcurrentDrainer.java
@@ -10,8 +10,10 @@
 import edu.mit.streamjit.impl.blob.Blob;
 import edu.mit.streamjit.impl.blob.Blob.Token;
 import edu.mit.streamjit.impl.common.BlobThread;
+import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.common.drainer.BlobGraph;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.common.CTRLRDrainElement.DrainType;
 import edu.mit.streamjit.impl.distributed.common.Utils;
 
@@ -34,8 +36,9 @@ public final class ConcurrentDrainer extends AbstractDrainer {
 	 */
 	ImmutableMap<Blob, Set<BlobThread>> threadMap;
 
-	public ConcurrentDrainer(BlobGraph blobGraph,
-			Map<Blob, Set<BlobThread>> threadMap) {
+	public ConcurrentDrainer(StreamJitApp<?, ?> app, TimeLogger logger,
+			BlobGraph blobGraph, Map<Blob, Set<BlobThread>> threadMap) {
+		super(app, logger);
 		setBlobGraph(blobGraph);
 		blobMap = buildBlobMap(threadMap.keySet());
 		this.threadMap = ImmutableMap.copyOf(threadMap);
@@ -98,6 +101,5 @@ public void run() {
 
 	@Override
 	protected void prepareDraining(boolean isFinal) {
-		// TODO Auto-generated method stub
 	}
 }

From 1185bcb9fd9809ba7631b23221f813977e414242 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sat, 7 Mar 2015 13:52:09 +0800
Subject: [PATCH 841/881] OnlineTuner&related classes moved to tuner package

OnlineTuner.java needs more refactoring. So I moved it to tuner package
before refactoring it.
---
 src/edu/mit/streamjit/impl/common/TimeLogger.java           | 2 +-
 .../mit/streamjit/impl/common/drainer/AbstractDrainer.java  | 2 +-
 .../streamjit/impl/distributed/ConfigurationManager.java    | 2 +-
 .../impl/distributed/DistributedStreamCompiler.java         | 2 +-
 src/edu/mit/streamjit/impl/distributed/StreamJitApp.java    | 2 +-
 src/edu/mit/streamjit/impl/distributed/common/Options.java  | 2 +-
 .../runtimer => tuner}/ConfigurationPrognosticator.java     | 2 +-
 .../runtimer => tuner}/GraphPropertyPrognosticator.java     | 2 +-
 .../distributed/runtimer => tuner}/MethodTimeLogger.java    | 2 +-
 .../{impl/distributed/runtimer => tuner}/OnlineTuner.java   | 6 ++----
 10 files changed, 11 insertions(+), 13 deletions(-)
 rename src/edu/mit/streamjit/{impl/distributed/runtimer => tuner}/ConfigurationPrognosticator.java (96%)
 rename src/edu/mit/streamjit/{impl/distributed/runtimer => tuner}/GraphPropertyPrognosticator.java (99%)
 rename src/edu/mit/streamjit/{impl/distributed/runtimer => tuner}/MethodTimeLogger.java (99%)
 rename src/edu/mit/streamjit/{impl/distributed/runtimer => tuner}/OnlineTuner.java (98%)

diff --git a/src/edu/mit/streamjit/impl/common/TimeLogger.java b/src/edu/mit/streamjit/impl/common/TimeLogger.java
index 4cd35349..cc9b8b89 100644
--- a/src/edu/mit/streamjit/impl/common/TimeLogger.java
+++ b/src/edu/mit/streamjit/impl/common/TimeLogger.java
@@ -2,7 +2,7 @@
 
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.tuner.OnlineTuner;
 
 /**
  * Logs various time measurements for off line performance analysis. Controller
diff --git a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
index e2a73024..bd4e0386 100644
--- a/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
+++ b/src/edu/mit/streamjit/impl/common/drainer/AbstractDrainer.java
@@ -35,7 +35,7 @@
 import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.SNDrainElement.SNDrainedData;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.tuner.OnlineTuner;
 
 /**
  * Abstract drainer is to perform draining on a stream application. Both
diff --git a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
index de297756..2511b1d7 100644
--- a/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
+++ b/src/edu/mit/streamjit/impl/distributed/ConfigurationManager.java
@@ -8,7 +8,7 @@
 import edu.mit.streamjit.api.Worker;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.drainer.BlobGraph;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.tuner.OnlineTuner;
 
 public class ConfigurationManager {
 
diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index e113f340..30fd34f7 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -35,9 +35,9 @@
 import edu.mit.streamjit.impl.distributed.runtimer.CommunicationManager.CommunicationType;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
 import edu.mit.streamjit.impl.distributed.runtimer.DistributedDrainer;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
+import edu.mit.streamjit.tuner.OnlineTuner;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
index c52459f0..1dccb5b7 100644
--- a/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
+++ b/src/edu/mit/streamjit/impl/distributed/StreamJitApp.java
@@ -42,9 +42,9 @@
 import edu.mit.streamjit.impl.distributed.common.Utils;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.impl.distributed.runtimer.Controller;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
 import edu.mit.streamjit.impl.interp.ChannelFactory;
 import edu.mit.streamjit.impl.interp.Interpreter;
+import edu.mit.streamjit.tuner.OnlineTuner;
 import edu.mit.streamjit.util.Pair;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Options.java b/src/edu/mit/streamjit/impl/distributed/common/Options.java
index fe0494d5..ed028e2e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Options.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Options.java
@@ -15,7 +15,7 @@
 import edu.mit.streamjit.impl.distributed.TailChannels;
 import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel1;
 import edu.mit.streamjit.impl.distributed.TailChannels.BlockingTailChannel2;
-import edu.mit.streamjit.impl.distributed.runtimer.OnlineTuner;
+import edu.mit.streamjit.tuner.OnlineTuner;
 import edu.mit.streamjit.tuner.TCPTuner;
 
 /**
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java b/src/edu/mit/streamjit/tuner/ConfigurationPrognosticator.java
similarity index 96%
rename from src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
rename to src/edu/mit/streamjit/tuner/ConfigurationPrognosticator.java
index 39d0ef72..123cdf76 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/ConfigurationPrognosticator.java
+++ b/src/edu/mit/streamjit/tuner/ConfigurationPrognosticator.java
@@ -1,4 +1,4 @@
-package edu.mit.streamjit.impl.distributed.runtimer;
+package edu.mit.streamjit.tuner;
 
 import edu.mit.streamjit.impl.common.Configuration;
 
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java b/src/edu/mit/streamjit/tuner/GraphPropertyPrognosticator.java
similarity index 99%
rename from src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
rename to src/edu/mit/streamjit/tuner/GraphPropertyPrognosticator.java
index 0d54175a..34e4c9b6 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/GraphPropertyPrognosticator.java
+++ b/src/edu/mit/streamjit/tuner/GraphPropertyPrognosticator.java
@@ -1,4 +1,4 @@
-package edu.mit.streamjit.impl.distributed.runtimer;
+package edu.mit.streamjit.tuner;
 
 import java.io.File;
 import java.io.FileWriter;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java b/src/edu/mit/streamjit/tuner/MethodTimeLogger.java
similarity index 99%
rename from src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
rename to src/edu/mit/streamjit/tuner/MethodTimeLogger.java
index 652bdeee..eb4351cf 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/MethodTimeLogger.java
+++ b/src/edu/mit/streamjit/tuner/MethodTimeLogger.java
@@ -1,4 +1,4 @@
-package edu.mit.streamjit.impl.distributed.runtimer;
+package edu.mit.streamjit.tuner;
 
 import java.io.IOException;
 import java.io.OutputStream;
diff --git a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
similarity index 98%
rename from src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
rename to src/edu/mit/streamjit/tuner/OnlineTuner.java
index bbd0182a..2cb636dd 100644
--- a/src/edu/mit/streamjit/impl/distributed/runtimer/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -1,4 +1,4 @@
-package edu.mit.streamjit.impl.distributed.runtimer;
+package edu.mit.streamjit.tuner;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -24,9 +24,7 @@
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.node.StreamNode;
-import edu.mit.streamjit.impl.distributed.runtimer.MethodTimeLogger.FileMethodTimeLogger;
-import edu.mit.streamjit.tuner.OpenTuner;
-import edu.mit.streamjit.tuner.TCPTuner;
+import edu.mit.streamjit.tuner.MethodTimeLogger.FileMethodTimeLogger;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.TimeLogProcessor;

From 64ac94f8fa5cf8eed9582ddac24890e38393b61e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Sun, 8 Mar 2015 18:57:01 +0800
Subject: [PATCH 842/881] reportingTime() writes cuttent count also

More discriptive message.
---
 src/edu/mit/streamjit/impl/distributed/TailChannels.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/TailChannels.java b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
index 020a235a..b4135d82 100644
--- a/src/edu/mit/streamjit/impl/distributed/TailChannels.java
+++ b/src/edu/mit/streamjit/impl/distributed/TailChannels.java
@@ -275,8 +275,9 @@ protected long unnormalizedTime(long time) {
 		 */
 		protected void reportingTime(long time) {
 			if (outputCountPrinter != null) {
-				String msg = String.format("Reporting...%s.cfg,time=%d\n",
-						cfgPrefix, time);
+				String msg = String.format(
+						"Reporting-%s.cfg,time=%d,TotalCount=%d\n", cfgPrefix,
+						time, count);
 				outputCountPrinter.write(msg);
 			}
 		}

From 77cecd926fff742994a4b960d4758fb6b9354923 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Mar 2015 19:06:59 +0800
Subject: [PATCH 843/881] BugFix:Always Creates ParititionManager.defaultCfg

Even if noOfNodes is one.
---
 .../impl/distributed/DistributedBlobFactory.java          | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
index c105ee4f..d565311d 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedBlobFactory.java
@@ -65,12 +65,8 @@ public Blob makeBlob(Set<Worker<?, ?>> workers, Configuration config,
 
 	@Override
 	public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
-		Configuration distCfg;
-		if (this.noOfMachines > 1)
-			distCfg = partitionManager.getDefaultConfiguration(workers, noOfMachines);
-		else
-			distCfg = Configuration.builder().build();
-
+		Configuration distCfg = partitionManager.getDefaultConfiguration(
+				workers, noOfMachines);
 		if (!Options.useCompilerBlob)
 			return distCfg;
 

From 1e2ef6f939868ef520c74a752bfaa8b7950715a8 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Mon, 9 Mar 2015 21:57:48 +0800
Subject: [PATCH 844/881] Ant:*.orig added to download script.

---
 jarapp.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jarapp.xml b/jarapp.xml
index 4686f7c8..c87cff99 100644
--- a/jarapp.xml
+++ b/jarapp.xml
@@ -144,7 +144,7 @@ fi
 mkdir -p $app
 cd $app
 mkdir -p $mainClass
-scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,streamgraph.dot\} $mainClass/
+scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/$mainClass/\{summary,*.txt,*.orig,streamgraph.dot\} $mainClass/
 scp -r ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app/\{*.sh,slurm-*,options.properties\} .
 #to download everything.
 #rsync -avh --progress ${username}@lanka.csail.mit.edu:/data/scratch/${username}/$app .

From b0f01100f932e792e80e9da2cf52b4db387adbdb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 12:23:08 +0800
Subject: [PATCH 845/881] Move all files to appDir/tune directory.

---
 .../impl/distributed/common/Utils.java        | 49 ++++++++++++++++++-
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index 7a4f31ef..acedf27e 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -1,11 +1,16 @@
 package edu.mit.streamjit.impl.distributed.common;
 
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
 import java.io.File;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryUsage;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
@@ -133,7 +138,7 @@ public static boolean createAppDir(String appName) {
 	 */
 	public static void writeReadMeTxt(String appName) {
 		try {
-			rename(appName, "README.txt");
+			// rename(appName, "README.txt");
 			FileWriter writer = new FileWriter(String.format("%s%sREADME.txt",
 					appName, File.separator));
 			DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
@@ -270,9 +275,49 @@ public static void backup(String appName) {
 		rename(appName, "profile.txt");
 	}
 
+	/**
+	 * Move all files and directories, except the configuration directory, from
+	 * appDir to appDir/tune directory. Does nothing if tune directory exists.
+	 * 
+	 * @param appName
+	 */
+	public static void backup1(String appName) {
+		File[] listOfFilesMove = listOfFilesMove(appName);
+		if (listOfFilesMove.length == 0)
+			return;
+
+		File tuneDir = new File(String.format("%s%stune", appName,
+				File.separator));
+		if (tuneDir.exists())
+			return;
+
+		if (!createDir(tuneDir.getPath()))
+			System.err.println(String.format("Creating %s dir failed.",
+					tuneDir.getPath()));
+		for (File f : listOfFilesMove) {
+			try {
+				Files.move(f.toPath(),
+						Paths.get(tuneDir.getPath(), f.getName()),
+						REPLACE_EXISTING);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	public static File[] listOfFilesMove(final String appName) {
+		File dir = new File(appName);
+		File[] files = dir.listFiles(new FilenameFilter() {
+			public boolean accept(File dir, String name) {
+				return !name.equals("configurations");
+			}
+		});
+		return files;
+	}
+
 	public static void newApp(String appName) {
 		createAppDir(appName);
-		backup(appName);
+		backup1(appName);
 		Utils.writeReadMeTxt(appName);
 	}
 }

From 9d7c0e1e9b7765dc95db108084711e436aaad570 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 12:36:18 +0800
Subject: [PATCH 846/881] ConfigurationUtils.configDir added

All classes gets the configuration directory name from the constant
ConfigurationUtils.configDir.
---
 .../streamjit/impl/common/ConfigurationAnalyzer.java   |  4 ++--
 .../mit/streamjit/impl/common/ConfigurationEditor.java |  4 ++--
 src/edu/mit/streamjit/impl/distributed/Visualizer.java |  4 ++--
 .../mit/streamjit/impl/distributed/common/Utils.java   |  6 +++---
 src/edu/mit/streamjit/util/ConfigurationUtils.java     | 10 ++++++----
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
index b10f197f..6150a2d3 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationAnalyzer.java
@@ -46,11 +46,11 @@ public static void main(String[] args) {
 	 */
 
 	public ConfigurationAnalyzer(String appName) {
-		verifyPath("configurations", appName);
+		verifyPath(ConfigurationUtils.configDir, appName);
 		bestConfigurations = new LinkedList<>();
 		this.appName = appName;
 		this.cfgDir = String.format("%s%s%s", appName, File.separator,
-				"configurations");
+				ConfigurationUtils.configDir);
 	}
 
 	private void compare(FloatParameter p1, FloatParameter p2) {
diff --git a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
index eb3527a3..948103b2 100644
--- a/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
+++ b/src/edu/mit/streamjit/impl/common/ConfigurationEditor.java
@@ -233,8 +233,8 @@ private static void print(String cfgFilePath) {
 	private static void convert() {
 		String appName = "ChannelVocoder7Kernel";
 		Configuration cfg = ConfigurationUtils.readConfiguration(String.format(
-				"%s%sconfigurations%s%d_%s.cfg", appName, File.separator,
-				File.separator, 1, appName));
+				"%s%s%s%s%d_%s.cfg", appName, File.separator,
+				ConfigurationUtils.configDir, File.separator, 1, appName));
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(
 					String.format("%d_%s.cfg", 0, appName)));
diff --git a/src/edu/mit/streamjit/impl/distributed/Visualizer.java b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
index a52d0081..a52e5096 100644
--- a/src/edu/mit/streamjit/impl/distributed/Visualizer.java
+++ b/src/edu/mit/streamjit/impl/distributed/Visualizer.java
@@ -214,8 +214,8 @@ private void runDot(String file) {
 			String outFileFormat = "svg";
 			String fileName = String.format("./%s%s%s.dot", appName,
 					File.separator, file);
-			String outFileName = String.format(
-					"./%s%sconfigurations%s%s_%s.%s", appName, File.separator,
+			String outFileName = String.format("./%s%s%s%s%s_%s.%s", appName,
+					File.separator, ConfigurationUtils.configDir,
 					File.separator, namePrefix, file, outFileFormat);
 			ProcessBuilder pb = new ProcessBuilder("dot", "-T" + outFileFormat,
 					fileName, "-o", outFileName);
diff --git a/src/edu/mit/streamjit/impl/distributed/common/Utils.java b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
index acedf27e..ca580a25 100644
--- a/src/edu/mit/streamjit/impl/distributed/common/Utils.java
+++ b/src/edu/mit/streamjit/impl/distributed/common/Utils.java
@@ -125,8 +125,8 @@ public static boolean createDir(String name) {
 	 */
 	public static boolean createAppDir(String appName) {
 		if (createDir(appName))
-			return createDir(String.format("%s%sconfigurations", appName,
-					File.separator));
+			return createDir(String.format("%s%s%s", appName, File.separator,
+					ConfigurationUtils.configDir));
 		else
 			return false;
 	}
@@ -309,7 +309,7 @@ public static File[] listOfFilesMove(final String appName) {
 		File dir = new File(appName);
 		File[] files = dir.listFiles(new FilenameFilter() {
 			public boolean accept(File dir, String name) {
-				return !name.equals("configurations");
+				return !name.equals(ConfigurationUtils.configDir);
 			}
 		});
 		return files;
diff --git a/src/edu/mit/streamjit/util/ConfigurationUtils.java b/src/edu/mit/streamjit/util/ConfigurationUtils.java
index 51b4525a..89c0cd3d 100644
--- a/src/edu/mit/streamjit/util/ConfigurationUtils.java
+++ b/src/edu/mit/streamjit/util/ConfigurationUtils.java
@@ -20,6 +20,8 @@
  */
 public class ConfigurationUtils {
 
+	public static final String configDir = "configurations";
+
 	/**
 	 * Reads configuration from ./appName/configurations/namePrefix_appName.cfg
 	 * and returns it.
@@ -37,8 +39,8 @@ public static Configuration readConfiguration(String appName,
 			String namePrefix) {
 		checkNotNull(appName);
 		namePrefix = namePrefix == null ? "" : namePrefix;
-		String cfgFilePath = String.format("%s%sconfigurations%s%s_%s.cfg",
-				appName, File.separator, File.separator, namePrefix, appName);
+		String cfgFilePath = String.format("%s%s%s%s%s_%s.cfg", appName,
+				File.separator, configDir, File.separator, namePrefix, appName);
 		return readConfiguration(cfgFilePath);
 	}
 
@@ -103,8 +105,8 @@ public static void saveConfg(String configJson, String namePrefix,
 			String appName) {
 		try {
 
-			File dir = new File(String.format("%s%sconfigurations", appName,
-					File.separator));
+			File dir = new File(String.format("%s%s%s", appName,
+					File.separator, configDir));
 			if (!dir.exists())
 				if (!dir.mkdirs()) {
 					System.err.println("Make directory failed");

From 3f02b69db62acff66369c3f96e89bfadecb57b37 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 13:28:39 +0800
Subject: [PATCH 847/881] Generates the graphs in pdf format.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 0aefd58a..37df3165 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -219,10 +219,16 @@ public static void summarize(String appName) throws IOException {
 
 	private static File makePlotFile(File dir, String name, String dataFile)
 			throws IOException {
+		boolean pdf = true;
 		File plotfile = new File(dir, "plot.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
-		writer.write("set terminal postscript eps enhanced color\n");
-		writer.write(String.format("set output \"%s.eps\"\n", name));
+		if (pdf) {
+			writer.write("set terminal pdf enhanced color\n");
+			writer.write(String.format("set output \"%s.pdf\"\n", name));
+		} else {
+			writer.write("set terminal postscript eps enhanced color\n");
+			writer.write(String.format("set output \"%s.eps\"\n", name));
+		}
 		writer.write("set ylabel \"Time(ms)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
 		writer.write(String.format("set title \"%s\"\n", name));
@@ -249,10 +255,16 @@ private static File makePlotFile(File dir, String name, String dataFile)
 
 	private static File makeHeapPlotFile(File dir, String name,
 			String dataFile1, String dataFile2) throws IOException {
+		boolean pdf = true;
 		File plotfile = new File(dir, "heapplot.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
-		writer.write("set terminal postscript eps enhanced color\n");
-		writer.write(String.format("set output \"%sHeap.eps\"\n", name));
+		if (pdf) {
+			writer.write("set terminal pdf enhanced color\n");
+			writer.write(String.format("set output \"%sHeap.pdf\"\n", name));
+		} else {
+			writer.write("set terminal postscript eps enhanced color\n");
+			writer.write(String.format("set output \"%sHeap.eps\"\n", name));
+		}
 		writer.write("set ylabel \"Memory(MB)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
 		writer.write(String.format("set title \"%sHeap\"\n", name));

From c8dd836e2d765ac118c5fc54a4c76c5b6235825b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 23:05:52 +0800
Subject: [PATCH 848/881] OnlineTuner refactored & Reconfigurer introduced.

First step toward refactoring OnlineTuner.java
---
 .../DistributedStreamCompiler.java            |  6 ++-
 src/edu/mit/streamjit/tuner/OnlineTuner.java  | 14 +++----
 src/edu/mit/streamjit/tuner/Reconfigurer.java | 38 +++++++++++++++++++
 3 files changed, 48 insertions(+), 10 deletions(-)
 create mode 100644 src/edu/mit/streamjit/tuner/Reconfigurer.java

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index 30fd34f7..bd660118 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -38,6 +38,7 @@
 import edu.mit.streamjit.partitioner.HorizontalPartitioner;
 import edu.mit.streamjit.partitioner.Partitioner;
 import edu.mit.streamjit.tuner.OnlineTuner;
+import edu.mit.streamjit.tuner.Reconfigurer;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
@@ -137,8 +138,9 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		CompiledStream cs = new DistributedCompiledStream(drainer);
 
 		if (Options.tune > 0 && this.cfg != null) {
-			OnlineTuner tuner = new OnlineTuner(drainer, manager, app,
-					cfgManager, logger, needTermination);
+			Reconfigurer configurer = new Reconfigurer(drainer, manager, app,
+					cfgManager, logger);
+			OnlineTuner tuner = new OnlineTuner(configurer, needTermination);
 			new Thread(tuner, "OnlineTuner").start();
 		}
 		return cs;
diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 2cb636dd..81d55e12 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -48,16 +48,14 @@ public class OnlineTuner implements Runnable {
 	private final Verifier verifier;
 	private final MethodTimeLogger mLogger;
 
-	public OnlineTuner(AbstractDrainer drainer, StreamJitAppManager manager,
-			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
-			TimeLogger logger, boolean needTermination) {
-		this.drainer = drainer;
-		this.manager = manager;
-		this.app = app;
-		this.cfgManager = cfgManager;
+	public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
+		this.drainer = configurer.drainer;
+		this.manager = configurer.manager;
+		this.app = configurer.app;
+		this.cfgManager = configurer.cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
-		this.logger = logger;
+		this.logger = configurer.logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
 		this.verifier = new Verifier();
 		this.mLogger = new FileMethodTimeLogger(app.name);
diff --git a/src/edu/mit/streamjit/tuner/Reconfigurer.java b/src/edu/mit/streamjit/tuner/Reconfigurer.java
new file mode 100644
index 00000000..4a7bb2e8
--- /dev/null
+++ b/src/edu/mit/streamjit/tuner/Reconfigurer.java
@@ -0,0 +1,38 @@
+package edu.mit.streamjit.tuner;
+
+import edu.mit.streamjit.impl.common.TimeLogger;
+import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
+import edu.mit.streamjit.impl.distributed.ConfigurationManager;
+import edu.mit.streamjit.impl.distributed.StreamJitApp;
+import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
+import edu.mit.streamjit.tuner.MethodTimeLogger.FileMethodTimeLogger;
+
+/**
+ * Re-factored the {@link OnlineTuner} and moved all streamjit app
+ * reconfiguration related methods to this new class.
+ * 
+ * @author sumanan
+ * @since 10 Mar, 2015
+ */
+public class Reconfigurer {
+
+	final AbstractDrainer drainer;
+	final StreamJitAppManager manager;
+	final StreamJitApp<?, ?> app;
+	final ConfigurationManager cfgManager;
+	final TimeLogger logger;
+	final ConfigurationPrognosticator prognosticator;
+	final MethodTimeLogger mLogger;
+
+	public Reconfigurer(AbstractDrainer drainer, StreamJitAppManager manager,
+			StreamJitApp<?, ?> app, ConfigurationManager cfgManager,
+			TimeLogger logger) {
+		this.drainer = drainer;
+		this.manager = manager;
+		this.app = app;
+		this.cfgManager = cfgManager;
+		this.logger = logger;
+		this.prognosticator = new GraphPropertyPrognosticator(app);
+		this.mLogger = new FileMethodTimeLogger(app.name);
+	}
+}
\ No newline at end of file

From fae80276a2524c6323dadbd9d86933aea1eadd0c Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 23:10:21 +0800
Subject: [PATCH 849/881] reconfigure() has been moved to Reconfigurer class

from OnlineTuner class.
---
 src/edu/mit/streamjit/tuner/OnlineTuner.java  | 104 +-----------------
 src/edu/mit/streamjit/tuner/Reconfigurer.java | 100 +++++++++++++++++
 2 files changed, 105 insertions(+), 99 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 81d55e12..0dd8086b 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -15,7 +15,6 @@
 import com.google.common.base.Stopwatch;
 
 import edu.mit.streamjit.impl.common.Configuration;
-import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
@@ -23,7 +22,6 @@
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Options;
-import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.MethodTimeLogger.FileMethodTimeLogger;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
@@ -47,8 +45,10 @@ public class OnlineTuner implements Runnable {
 	private final ConfigurationPrognosticator prognosticator;
 	private final Verifier verifier;
 	private final MethodTimeLogger mLogger;
+	private final Reconfigurer configurer;
 
 	public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
+		this.configurer = configurer;
 		this.drainer = configurer.drainer;
 		this.manager = configurer.manager;
 		this.app = configurer.app;
@@ -113,7 +113,7 @@ private void tune() {
 				Configuration config = newCfg(++round, cfgJson);
 				mLogger.eNewCfg(round);
 				mLogger.bReconfigure();
-				ret = reconfigure(config, 2 * currentBestTime);
+				ret = configurer.reconfigure(config, 2 * currentBestTime);
 				mLogger.eReconfigure();
 				if (ret.first) {
 					long time = ret.second;
@@ -159,100 +159,6 @@ private void startTuner() throws IOException {
 		tuner.writeLine(Jsonifiers.toJson(app.getConfiguration()).toString());
 	}
 
-	/**
-	 * TODO: Split this method into two methods, 1.reconfigure(),
-	 * 2.getFixedOutputTime().
-	 * 
-	 * @param cfgJson
-	 * @param round
-	 * @return if ret.first == false, then no more tuning. ret.second = running
-	 *         time in milliseconds. ret.second may be a negative value if the
-	 *         reconfiguration is unsuccessful or a timeout is occurred.
-	 *         Meanings of the negative values are follows
-	 *         <ol>
-	 *         <li>-1: Timeout has occurred.
-	 *         <li>-2: Invalid configuration.
-	 *         <li>-3: {@link ConfigurationPrognosticator} has rejected the
-	 *         configuration.
-	 *         <li>-4: Draining failed. Another draining is in progress.
-	 *         <li>-5: Reconfiguration has failed at {@link StreamNode} side.
-	 *         E.g., Compilation error.
-	 *         <li>-6: Misc problems.
-	 */
-	private Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
-		long time;
-
-		if (manager.getStatus() == AppStatus.STOPPED)
-			return new Pair<Boolean, Long>(false, 0l);
-
-		mLogger.bCfgManagerNewcfg();
-		boolean validCfg = cfgManager.newConfiguration(config);
-		mLogger.eCfgManagerNewcfg();
-		if (!validCfg)
-			return new Pair<Boolean, Long>(true, -2l);
-
-		mLogger.bPrognosticate();
-		boolean prog = prognosticator.prognosticate(config);
-		mLogger.ePrognosticate();
-		if (!prog)
-			return new Pair<Boolean, Long>(true, -3l);
-
-		try {
-			mLogger.bIntermediateDraining();
-			boolean intermediateDraining = intermediateDraining();
-			mLogger.eIntermediateDraining();
-			if (!intermediateDraining)
-				return new Pair<Boolean, Long>(false, -4l);
-
-			drainer.setBlobGraph(app.blobGraph);
-			int multiplier = getMultiplier(config);
-			mLogger.bManagerReconfigure();
-			boolean reconfigure = manager.reconfigure(multiplier);
-			mLogger.eManagerReconfigure();
-			if (reconfigure) {
-				// TODO: need to check the manager's status before passing the
-				// time. Exceptions, final drain, etc may causes app to stop
-				// executing.
-				mLogger.bGetFixedOutputTime();
-				time = manager.getFixedOutputTime(timeout);
-				mLogger.eGetFixedOutputTime();
-				logger.logRunTime(time);
-			} else {
-				time = -5l;
-			}
-		} catch (Exception ex) {
-			ex.printStackTrace();
-			System.err
-					.println("Couldn't compile the stream graph with this configuration");
-			time = -6l;
-		}
-		return new Pair<Boolean, Long>(true, time);
-	}
-
-	/**
-	 * Performs intermediate draining.
-	 * 
-	 * @return <code>true</code> iff the draining is success or the application
-	 *         is not running currently.
-	 * @throws InterruptedException
-	 */
-	private boolean intermediateDraining() throws InterruptedException {
-		if (manager.isRunning()) {
-			return drainer.drainIntermediate();
-		} else
-			return true;
-	}
-
-	private int getMultiplier(Configuration config) {
-		int multiplier = 50;
-		IntParameter mulParam = config.getParameter("multiplier",
-				IntParameter.class);
-		if (mulParam != null)
-			multiplier = mulParam.getValue();
-		System.err.println("Reconfiguring...multiplier = " + multiplier);
-		return multiplier;
-	}
-
 	/**
 	 * Just excerpted from run() method for better readability.
 	 * 
@@ -270,7 +176,7 @@ private void handleTermination() throws IOException {
 		if (needTermination) {
 			terminate();
 		} else {
-			Pair<Boolean, Long> ret = reconfigure(finalcfg, 0);
+			Pair<Boolean, Long> ret = configurer.reconfigure(finalcfg, 0);
 			if (ret.first && ret.second > 0)
 				System.out
 						.println("Application is running forever with the final configuration.");
@@ -459,7 +365,7 @@ private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			if (cfg != null) {
 				for (int i = 0; i < count; i++) {
 					logger.newConfiguration(cfgName);
-					ret = reconfigure(cfg, 0);
+					ret = configurer.reconfigure(cfg, 0);
 					if (ret.first) {
 						prognosticator.time(ret.second);
 						runningTime.add(ret.second);
diff --git a/src/edu/mit/streamjit/tuner/Reconfigurer.java b/src/edu/mit/streamjit/tuner/Reconfigurer.java
index 4a7bb2e8..7a33435a 100644
--- a/src/edu/mit/streamjit/tuner/Reconfigurer.java
+++ b/src/edu/mit/streamjit/tuner/Reconfigurer.java
@@ -1,11 +1,16 @@
 package edu.mit.streamjit.tuner;
 
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.common.Configuration.IntParameter;
 import edu.mit.streamjit.impl.common.TimeLogger;
 import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
 import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
+import edu.mit.streamjit.impl.distributed.common.AppStatus;
+import edu.mit.streamjit.impl.distributed.node.StreamNode;
 import edu.mit.streamjit.tuner.MethodTimeLogger.FileMethodTimeLogger;
+import edu.mit.streamjit.util.Pair;
 
 /**
  * Re-factored the {@link OnlineTuner} and moved all streamjit app
@@ -35,4 +40,99 @@ public Reconfigurer(AbstractDrainer drainer, StreamJitAppManager manager,
 		this.prognosticator = new GraphPropertyPrognosticator(app);
 		this.mLogger = new FileMethodTimeLogger(app.name);
 	}
+
+	/**
+	 * TODO: Split this method into two methods, 1.reconfigure(),
+	 * 2.getFixedOutputTime().
+	 * 
+	 * @param cfgJson
+	 * @param round
+	 * @return if ret.first == false, then no more tuning. ret.second = running
+	 *         time in milliseconds. ret.second may be a negative value if the
+	 *         reconfiguration is unsuccessful or a timeout is occurred.
+	 *         Meanings of the negative values are follows
+	 *         <ol>
+	 *         <li>-1: Timeout has occurred.
+	 *         <li>-2: Invalid configuration.
+	 *         <li>-3: {@link ConfigurationPrognosticator} has rejected the
+	 *         configuration.
+	 *         <li>-4: Draining failed. Another draining is in progress.
+	 *         <li>-5: Reconfiguration has failed at {@link StreamNode} side.
+	 *         E.g., Compilation error.
+	 *         <li>-6: Misc problems.
+	 */
+	public Pair<Boolean, Long> reconfigure(Configuration config, long timeout) {
+		long time;
+
+		if (manager.getStatus() == AppStatus.STOPPED)
+			return new Pair<Boolean, Long>(false, 0l);
+
+		mLogger.bCfgManagerNewcfg();
+		boolean validCfg = cfgManager.newConfiguration(config);
+		mLogger.eCfgManagerNewcfg();
+		if (!validCfg)
+			return new Pair<Boolean, Long>(true, -2l);
+
+		mLogger.bPrognosticate();
+		boolean prog = prognosticator.prognosticate(config);
+		mLogger.ePrognosticate();
+		if (!prog)
+			return new Pair<Boolean, Long>(true, -3l);
+
+		try {
+			mLogger.bIntermediateDraining();
+			boolean intermediateDraining = intermediateDraining();
+			mLogger.eIntermediateDraining();
+			if (!intermediateDraining)
+				return new Pair<Boolean, Long>(false, -4l);
+
+			drainer.setBlobGraph(app.blobGraph);
+			int multiplier = getMultiplier(config);
+			mLogger.bManagerReconfigure();
+			boolean reconfigure = manager.reconfigure(multiplier);
+			mLogger.eManagerReconfigure();
+			if (reconfigure) {
+				// TODO: need to check the manager's status before passing the
+				// time. Exceptions, final drain, etc may causes app to stop
+				// executing.
+				mLogger.bGetFixedOutputTime();
+				time = manager.getFixedOutputTime(timeout);
+				mLogger.eGetFixedOutputTime();
+				logger.logRunTime(time);
+			} else {
+				time = -5l;
+			}
+		} catch (Exception ex) {
+			ex.printStackTrace();
+			System.err
+					.println("Couldn't compile the stream graph with this configuration");
+			time = -6l;
+		}
+		return new Pair<Boolean, Long>(true, time);
+	}
+
+	/**
+	 * Performs intermediate draining.
+	 * 
+	 * @return <code>true</code> iff the draining is success or the application
+	 *         is not running currently.
+	 * @throws InterruptedException
+	 */
+	private boolean intermediateDraining() throws InterruptedException {
+		if (manager.isRunning()) {
+			return drainer.drainIntermediate();
+		} else
+			return true;
+	}
+
+	private int getMultiplier(Configuration config) {
+		int multiplier = 50;
+		IntParameter mulParam = config.getParameter("multiplier",
+				IntParameter.class);
+		if (mulParam != null)
+			multiplier = mulParam.getValue();
+		System.err.println("Reconfiguring...multiplier = " + multiplier);
+		return multiplier;
+	}
+
 }
\ No newline at end of file

From c0f87295b7b305c04616418d8d2d15103461204e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Tue, 10 Mar 2015 23:15:49 +0800
Subject: [PATCH 850/881] terminate() has been moved to Reconfigurer class

from OnlineTuner class.
---
 src/edu/mit/streamjit/tuner/OnlineTuner.java  | 17 ++++-------------
 src/edu/mit/streamjit/tuner/Reconfigurer.java |  8 ++++++++
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 0dd8086b..3fbf8788 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -67,7 +67,7 @@ public void run() {
 			tune();
 		else if (Options.tune == 2) {
 			verifier.verify();
-			terminate();
+			configurer.terminate();
 		} else
 			System.err
 					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
@@ -133,7 +133,7 @@ private void tune() {
 		} catch (IOException e) {
 			e.printStackTrace();
 			mLogger.bTerminate();
-			terminate();
+			configurer.terminate();
 			mLogger.eTerminate();
 		}
 		mLogger.bTuningFinished();
@@ -174,7 +174,7 @@ private void handleTermination() throws IOException {
 		verifier.verify();
 
 		if (needTermination) {
-			terminate();
+			configurer.terminate();
 		} else {
 			Pair<Boolean, Long> ret = configurer.reconfigure(finalcfg, 0);
 			if (ret.first && ret.second > 0)
@@ -206,7 +206,7 @@ private void tuningFinished() {
 		}
 
 		if (needTermination)
-			terminate();
+			configurer.terminate();
 
 		try {
 			TimeLogProcessor.summarize(app.name);
@@ -215,15 +215,6 @@ private void tuningFinished() {
 		}
 	}
 
-	private void terminate() {
-		if (manager.isRunning()) {
-			// drainer.startDraining(1);
-			drainer.drainFinal(true);
-		} else {
-			manager.stop();
-		}
-	}
-
 	private class Verifier {
 
 		public void verify() {
diff --git a/src/edu/mit/streamjit/tuner/Reconfigurer.java b/src/edu/mit/streamjit/tuner/Reconfigurer.java
index 7a33435a..56d912ae 100644
--- a/src/edu/mit/streamjit/tuner/Reconfigurer.java
+++ b/src/edu/mit/streamjit/tuner/Reconfigurer.java
@@ -135,4 +135,12 @@ private int getMultiplier(Configuration config) {
 		return multiplier;
 	}
 
+	public void terminate() {
+		if (manager.isRunning()) {
+			// drainer.startDraining(1);
+			drainer.drainFinal(true);
+		} else {
+			manager.stop();
+		}
+	}
 }
\ No newline at end of file

From fc8059091d9ef853e220ca737877fd2b2f8d1069 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 08:07:30 +0800
Subject: [PATCH 851/881] Inner class Verifier has been made static inner

Verifier has been made independent from the outter class OnlineTuner.
Next step is to move this class Verifier to a new file.
---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 26 +++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 3fbf8788..e2fde0ec 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -57,7 +57,7 @@ public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
 		this.needTermination = needTermination;
 		this.logger = configurer.logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
-		this.verifier = new Verifier();
+		this.verifier = new Verifier(configurer);
 		this.mLogger = new FileMethodTimeLogger(app.name);
 	}
 
@@ -215,7 +215,15 @@ private void tuningFinished() {
 		}
 	}
 
-	private class Verifier {
+	private static class Verifier {
+
+		private final Reconfigurer configurer;
+		private final String appName;
+
+		Verifier(Reconfigurer configurer) {
+			this.configurer = configurer;
+			this.appName = configurer.app.name;
+		}
 
 		public void verify() {
 			verifyTuningTimes(cfgPrefixes());
@@ -238,9 +246,9 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 						String prefix = en.getKey();
 						Integer expectedRunningTime = en.getValue();
 						String cfgName = String.format("%s_%s.cfg", prefix,
-								app.name);
+								appName);
 						Configuration cfg = ConfigurationUtils
-								.readConfiguration(app.name, prefix);
+								.readConfiguration(appName, prefix);
 						if (cfg == null) {
 							System.err.println(String.format(
 									"No %s file exists", cfgName));
@@ -284,7 +292,7 @@ private void processRunningTimes(List<Long> runningTimes,
 
 		private FileWriter writer() throws IOException {
 			FileWriter writer = new FileWriter(String.format(
-					"%s%sevaluation.txt", app.name, File.separator, app.name),
+					"%s%sevaluation.txt", appName, File.separator, appName),
 					true);
 			writer.write("##########################################################");
 			Properties prop = Options.getProperties();
@@ -297,8 +305,8 @@ private Map<String, Integer> cfgPrefixes() {
 			cfgPrefixes.put("final", 0);
 			cfgPrefixes.put("hand", 0);
 			try {
-				BufferedReader reader = new BufferedReader(new FileReader(
-						String.format("%s%sverify.txt", app.name,
+				BufferedReader reader = new BufferedReader(
+						new FileReader(String.format("%s%sverify.txt", appName,
 								File.separator)));
 				String line;
 				while ((line = reader.readLine()) != null) {
@@ -355,10 +363,10 @@ private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 			Pair<Boolean, Long> ret;
 			if (cfg != null) {
 				for (int i = 0; i < count; i++) {
-					logger.newConfiguration(cfgName);
+					configurer.logger.newConfiguration(cfgName);
 					ret = configurer.reconfigure(cfg, 0);
 					if (ret.first) {
-						prognosticator.time(ret.second);
+						configurer.prognosticator.time(ret.second);
 						runningTime.add(ret.second);
 					} else {
 						System.err.println("Evaluation failed...");

From 068c6572cb05412b55f4cc17e178ee7c00c50798 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 08:14:12 +0800
Subject: [PATCH 852/881] Verifier has been moved to a new file.

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 169 ------------------
 src/edu/mit/streamjit/tuner/Verifier.java    | 178 +++++++++++++++++++
 2 files changed, 178 insertions(+), 169 deletions(-)
 create mode 100644 src/edu/mit/streamjit/tuner/Verifier.java

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index e2fde0ec..eb489fa6 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -1,15 +1,10 @@
 package edu.mit.streamjit.tuner;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
@@ -214,168 +209,4 @@ private void tuningFinished() {
 			e.printStackTrace();
 		}
 	}
-
-	private static class Verifier {
-
-		private final Reconfigurer configurer;
-		private final String appName;
-
-		Verifier(Reconfigurer configurer) {
-			this.configurer = configurer;
-			this.appName = configurer.app.name;
-		}
-
-		public void verify() {
-			verifyTuningTimes(cfgPrefixes());
-		}
-
-		/**
-		 * This method just picks a few configurations and re-run the app to
-		 * ensure the time we reported to the opentuner is correct.
-		 * 
-		 * This method can be called after the completion of the tuning.
-		 * 
-		 * @param cfgPrefixes
-		 *            map of cfgPrefixes and expected running time.
-		 */
-		private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
-			try {
-				FileWriter writer = writer();
-				for (int i = 0; i < Options.verificationCount; i++) {
-					for (Map.Entry<String, Integer> en : cfgPrefixes.entrySet()) {
-						String prefix = en.getKey();
-						Integer expectedRunningTime = en.getValue();
-						String cfgName = String.format("%s_%s.cfg", prefix,
-								appName);
-						Configuration cfg = ConfigurationUtils
-								.readConfiguration(appName, prefix);
-						if (cfg == null) {
-							System.err.println(String.format(
-									"No %s file exists", cfgName));
-							continue;
-						}
-						cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
-						writer.write("----------------------------------------\n");
-						writer.write(String.format("Configuration name = %s\n",
-								cfgName));
-						List<Long> runningTimes = evaluateConfig(cfg, cfgName);
-						processRunningTimes(runningTimes, expectedRunningTime,
-								writer);
-					}
-				}
-				writer.write("**************FINISHED**************\n\n");
-				writer.close();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-
-		private void processRunningTimes(List<Long> runningTimes,
-				Integer expectedRunningTime, FileWriter writer)
-				throws IOException {
-			writer.write(String.format("Expected running time = %dms\n",
-					expectedRunningTime));
-			int correctEval = 0;
-			double total = 0;
-			for (int i = 0; i < runningTimes.size(); i++) {
-				long runningTime = runningTimes.get(i);
-				if (runningTime > 0) {
-					correctEval++;
-					total += runningTime;
-				}
-				writer.write(String.format("Evaluation %d = %dms\n", i + 1,
-						runningTime));
-			}
-			double avg = total / correctEval;
-			writer.write(String.format("Average running time = %.3fms\n", avg));
-		}
-
-		private FileWriter writer() throws IOException {
-			FileWriter writer = new FileWriter(String.format(
-					"%s%sevaluation.txt", appName, File.separator, appName),
-					true);
-			writer.write("##########################################################");
-			Properties prop = Options.getProperties();
-			prop.store(writer, "");
-			return writer;
-		}
-
-		private Map<String, Integer> cfgPrefixes() {
-			Map<String, Integer> cfgPrefixes = new HashMap<>();
-			cfgPrefixes.put("final", 0);
-			cfgPrefixes.put("hand", 0);
-			try {
-				BufferedReader reader = new BufferedReader(
-						new FileReader(String.format("%s%sverify.txt", appName,
-								File.separator)));
-				String line;
-				while ((line = reader.readLine()) != null) {
-					if (line.contains("="))
-						process1(line, cfgPrefixes);
-					else
-						process2(line, cfgPrefixes);
-				}
-				reader.close();
-			} catch (IOException e) {
-			}
-			return cfgPrefixes;
-		}
-
-		/**
-		 * Processes the line that is generated by {@link TimeLogProcessor}.
-		 */
-		private void process1(String line, Map<String, Integer> cfgPrefixes) {
-			String[] arr = line.split("=");
-			String cfgPrefix = arr[0].trim();
-			int expectedTime = 0;
-			if (arr.length > 1)
-				try {
-					expectedTime = Integer.parseInt(arr[1]);
-				} catch (NumberFormatException ex) {
-					System.err.println("NumberFormatException: " + arr[1]);
-				}
-			cfgPrefixes.put(cfgPrefix, expectedTime);
-		}
-
-		/**
-		 * Processes manually entered lines in the verify.txt
-		 */
-		private void process2(String line, Map<String, Integer> cfgPrefixes) {
-			String[] arr = line.split(",");
-			for (String s : arr) {
-				cfgPrefixes.put(s.trim(), 0);
-			}
-		}
-
-		/**
-		 * Evaluates a configuration.
-		 * 
-		 * @param cfg
-		 *            configuration that needs to be evaluated
-		 * @param cfgName
-		 *            name of the configuration. This is just for logging
-		 *            purpose.
-		 */
-		private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
-			System.out.println("Evaluating " + cfgName);
-			int count = Options.evaluationCount;
-			List<Long> runningTime = new ArrayList<>(count);
-			Pair<Boolean, Long> ret;
-			if (cfg != null) {
-				for (int i = 0; i < count; i++) {
-					configurer.logger.newConfiguration(cfgName);
-					ret = configurer.reconfigure(cfg, 0);
-					if (ret.first) {
-						configurer.prognosticator.time(ret.second);
-						runningTime.add(ret.second);
-					} else {
-						System.err.println("Evaluation failed...");
-					}
-				}
-			} else {
-				System.err.println("Null configuration\n");
-			}
-			return runningTime;
-		}
-	}
 }
\ No newline at end of file
diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
new file mode 100644
index 00000000..e38726eb
--- /dev/null
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -0,0 +1,178 @@
+package edu.mit.streamjit.tuner;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import edu.mit.streamjit.impl.common.Configuration;
+import edu.mit.streamjit.impl.distributed.common.Options;
+import edu.mit.streamjit.util.ConfigurationUtils;
+import edu.mit.streamjit.util.Pair;
+import edu.mit.streamjit.util.TimeLogProcessor;
+
+public class Verifier {
+
+	private final Reconfigurer configurer;
+	private final String appName;
+
+	Verifier(Reconfigurer configurer) {
+		this.configurer = configurer;
+		this.appName = configurer.app.name;
+	}
+
+	public void verify() {
+		verifyTuningTimes(cfgPrefixes());
+	}
+
+	/**
+	 * This method just picks a few configurations and re-run the app to ensure
+	 * the time we reported to the opentuner is correct.
+	 * 
+	 * This method can be called after the completion of the tuning.
+	 * 
+	 * @param cfgPrefixes
+	 *            map of cfgPrefixes and expected running time.
+	 */
+	private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
+		try {
+			FileWriter writer = writer();
+			for (int i = 0; i < Options.verificationCount; i++) {
+				for (Map.Entry<String, Integer> en : cfgPrefixes.entrySet()) {
+					String prefix = en.getKey();
+					Integer expectedRunningTime = en.getValue();
+					String cfgName = String
+							.format("%s_%s.cfg", prefix, appName);
+					Configuration cfg = ConfigurationUtils.readConfiguration(
+							appName, prefix);
+					if (cfg == null) {
+						System.err.println(String.format("No %s file exists",
+								cfgName));
+						continue;
+					}
+					cfg = ConfigurationUtils.addConfigPrefix(cfg, prefix);
+					writer.write("----------------------------------------\n");
+					writer.write(String.format("Configuration name = %s\n",
+							cfgName));
+					List<Long> runningTimes = evaluateConfig(cfg, cfgName);
+					processRunningTimes(runningTimes, expectedRunningTime,
+							writer);
+				}
+			}
+			writer.write("**************FINISHED**************\n\n");
+			writer.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	private void processRunningTimes(List<Long> runningTimes,
+			Integer expectedRunningTime, FileWriter writer) throws IOException {
+		writer.write(String.format("Expected running time = %dms\n",
+				expectedRunningTime));
+		int correctEval = 0;
+		double total = 0;
+		for (int i = 0; i < runningTimes.size(); i++) {
+			long runningTime = runningTimes.get(i);
+			if (runningTime > 0) {
+				correctEval++;
+				total += runningTime;
+			}
+			writer.write(String.format("Evaluation %d = %dms\n", i + 1,
+					runningTime));
+		}
+		double avg = total / correctEval;
+		writer.write(String.format("Average running time = %.3fms\n", avg));
+	}
+
+	private FileWriter writer() throws IOException {
+		FileWriter writer = new FileWriter(String.format("%s%sevaluation.txt",
+				appName, File.separator, appName), true);
+		writer.write("##########################################################");
+		Properties prop = Options.getProperties();
+		prop.store(writer, "");
+		return writer;
+	}
+
+	private Map<String, Integer> cfgPrefixes() {
+		Map<String, Integer> cfgPrefixes = new HashMap<>();
+		cfgPrefixes.put("final", 0);
+		cfgPrefixes.put("hand", 0);
+		try {
+			BufferedReader reader = new BufferedReader(new FileReader(
+					String.format("%s%sverify.txt", appName, File.separator)));
+			String line;
+			while ((line = reader.readLine()) != null) {
+				if (line.contains("="))
+					process1(line, cfgPrefixes);
+				else
+					process2(line, cfgPrefixes);
+			}
+			reader.close();
+		} catch (IOException e) {
+		}
+		return cfgPrefixes;
+	}
+
+	/**
+	 * Processes the line that is generated by {@link TimeLogProcessor}.
+	 */
+	private void process1(String line, Map<String, Integer> cfgPrefixes) {
+		String[] arr = line.split("=");
+		String cfgPrefix = arr[0].trim();
+		int expectedTime = 0;
+		if (arr.length > 1)
+			try {
+				expectedTime = Integer.parseInt(arr[1]);
+			} catch (NumberFormatException ex) {
+				System.err.println("NumberFormatException: " + arr[1]);
+			}
+		cfgPrefixes.put(cfgPrefix, expectedTime);
+	}
+
+	/**
+	 * Processes manually entered lines in the verify.txt
+	 */
+	private void process2(String line, Map<String, Integer> cfgPrefixes) {
+		String[] arr = line.split(",");
+		for (String s : arr) {
+			cfgPrefixes.put(s.trim(), 0);
+		}
+	}
+
+	/**
+	 * Evaluates a configuration.
+	 * 
+	 * @param cfg
+	 *            configuration that needs to be evaluated
+	 * @param cfgName
+	 *            name of the configuration. This is just for logging purpose.
+	 */
+	private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
+		System.out.println("Evaluating " + cfgName);
+		int count = Options.evaluationCount;
+		List<Long> runningTime = new ArrayList<>(count);
+		Pair<Boolean, Long> ret;
+		if (cfg != null) {
+			for (int i = 0; i < count; i++) {
+				configurer.logger.newConfiguration(cfgName);
+				ret = configurer.reconfigure(cfg, 0);
+				if (ret.first) {
+					configurer.prognosticator.time(ret.second);
+					runningTime.add(ret.second);
+				} else {
+					System.err.println("Evaluation failed...");
+				}
+			}
+		} else {
+			System.err.println("Null configuration\n");
+		}
+		return runningTime;
+	}
+}
\ No newline at end of file

From 8e3187f87bdc87196d7ddd66c665dd61f2a26e32 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 08:35:20 +0800
Subject: [PATCH 853/881] Verifier implements Runnable.

---
 src/edu/mit/streamjit/tuner/Verifier.java | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index e38726eb..973b5d11 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -17,12 +17,12 @@
 import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.TimeLogProcessor;
 
-public class Verifier {
+public class Verifier implements Runnable {
 
 	private final Reconfigurer configurer;
 	private final String appName;
 
-	Verifier(Reconfigurer configurer) {
+	public Verifier(Reconfigurer configurer) {
 		this.configurer = configurer;
 		this.appName = configurer.app.name;
 	}
@@ -175,4 +175,13 @@ private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
 		}
 		return runningTime;
 	}
+
+	@Override
+	public void run() {
+		if (Options.tune == 2) {
+			verify();
+			configurer.terminate();
+		} else
+			System.err.println("Options.tune is not in the verify mode");
+	}
 }
\ No newline at end of file

From 81096d7b8381846c91efdc5ea02cfae92bc4ccee Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 08:35:58 +0800
Subject: [PATCH 854/881] DSC starts Tuner/Verifier based on Options.tune

---
 .../distributed/DistributedStreamCompiler.java | 18 ++++++++++++++++--
 src/edu/mit/streamjit/tuner/OnlineTuner.java   |  8 ++------
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
index bd660118..c4713502 100644
--- a/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
+++ b/src/edu/mit/streamjit/impl/distributed/DistributedStreamCompiler.java
@@ -39,6 +39,7 @@
 import edu.mit.streamjit.partitioner.Partitioner;
 import edu.mit.streamjit.tuner.OnlineTuner;
 import edu.mit.streamjit.tuner.Reconfigurer;
+import edu.mit.streamjit.tuner.Verifier;
 import edu.mit.streamjit.util.ConfigurationUtils;
 
 /**
@@ -140,8 +141,7 @@ public <I, O> CompiledStream compile(OneToOneElement<I, O> stream,
 		if (Options.tune > 0 && this.cfg != null) {
 			Reconfigurer configurer = new Reconfigurer(drainer, manager, app,
 					cfgManager, logger);
-			OnlineTuner tuner = new OnlineTuner(configurer, needTermination);
-			new Thread(tuner, "OnlineTuner").start();
+			tuneOrVerify(configurer, needTermination);
 		}
 		return cs;
 	}
@@ -299,6 +299,20 @@ private <I, O> boolean verifyCfg(Configuration defaultCfg, Configuration cfg) {
 		return false;
 	}
 
+	private void tuneOrVerify(Reconfigurer configurer, boolean needTermination) {
+		Runnable r;
+		if (Options.tune == 1) {
+			r = new OnlineTuner(configurer, needTermination);
+			new Thread(r, "OnlineTuner").start();
+		} else if (Options.tune == 2) {
+			r = new Verifier(configurer);
+			new Thread(r, "Verifier").start();
+		} else {
+			throw new IllegalStateException(
+					"Neither OnlineTuner nor Verifer has been started.");
+		}
+	}
+
 	private static class DistributedCompiledStream implements CompiledStream {
 
 		AbstractDrainer drainer;
diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index eb489fa6..218b25c9 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -60,12 +60,8 @@ public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
 	public void run() {
 		if (Options.tune == 1)
 			tune();
-		else if (Options.tune == 2) {
-			verifier.verify();
-			configurer.terminate();
-		} else
-			System.err
-					.println("GlobalConstants.tune is neither in tune mode nor in evaluate mode.");
+		else
+			System.err.println("Options.tune is not in tune mode.");
 	}
 
 	private void tune() {

From 75bd7dc8e35c50b7e0174df7878ea955680d1229 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 08:39:33 +0800
Subject: [PATCH 855/881] Instance field OnlineTuner.verifer has been removed.

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 218b25c9..9df59b61 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -38,7 +38,6 @@ public class OnlineTuner implements Runnable {
 	private final boolean needTermination;
 	private final TimeLogger logger;
 	private final ConfigurationPrognosticator prognosticator;
-	private final Verifier verifier;
 	private final MethodTimeLogger mLogger;
 	private final Reconfigurer configurer;
 
@@ -52,7 +51,6 @@ public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
 		this.needTermination = needTermination;
 		this.logger = configurer.logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
-		this.verifier = new Verifier(configurer);
 		this.mLogger = new FileMethodTimeLogger(app.name);
 	}
 
@@ -162,7 +160,7 @@ private void handleTermination() throws IOException {
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
 
-		verifier.verify();
+		new Verifier(configurer).verify();
 
 		if (needTermination) {
 			configurer.terminate();

From b56058297c9b0d224b25601678f8a15f9cd590bd Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 09:17:53 +0800
Subject: [PATCH 856/881] OnlineTuner uses Reconfigurer's MethodTimeLogger.

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 9df59b61..2fb8bfd6 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -51,7 +51,7 @@ public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
 		this.needTermination = needTermination;
 		this.logger = configurer.logger;
 		this.prognosticator = new GraphPropertyPrognosticator(app);
-		this.mLogger = new FileMethodTimeLogger(app.name);
+		this.mLogger = configurer.mLogger;
 	}
 
 	@Override

From 07c09b69c970f726d7fd3d11950d217cf5fe3575 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 09:29:24 +0800
Subject: [PATCH 857/881] OnlineTuner's few instance fields have been removed

Access those objects through Reconfigurer.
---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 2fb8bfd6..2681324c 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -11,13 +11,10 @@
 
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.TimeLogger;
-import edu.mit.streamjit.impl.common.drainer.AbstractDrainer;
 import edu.mit.streamjit.impl.distributed.ConfigurationManager;
 import edu.mit.streamjit.impl.distributed.StreamJitApp;
-import edu.mit.streamjit.impl.distributed.StreamJitAppManager;
 import edu.mit.streamjit.impl.distributed.common.AppStatus;
 import edu.mit.streamjit.impl.distributed.common.Options;
-import edu.mit.streamjit.tuner.MethodTimeLogger.FileMethodTimeLogger;
 import edu.mit.streamjit.util.ConfigurationUtils;
 import edu.mit.streamjit.util.Pair;
 import edu.mit.streamjit.util.TimeLogProcessor;
@@ -30,8 +27,6 @@
  * @since Oct 8, 2013
  */
 public class OnlineTuner implements Runnable {
-	private final AbstractDrainer drainer;
-	private final StreamJitAppManager manager;
 	private final OpenTuner tuner;
 	private final StreamJitApp<?, ?> app;
 	private final ConfigurationManager cfgManager;
@@ -43,14 +38,12 @@ public class OnlineTuner implements Runnable {
 
 	public OnlineTuner(Reconfigurer configurer, boolean needTermination) {
 		this.configurer = configurer;
-		this.drainer = configurer.drainer;
-		this.manager = configurer.manager;
 		this.app = configurer.app;
 		this.cfgManager = configurer.cfgManager;
 		this.tuner = new TCPTuner();
 		this.needTermination = needTermination;
 		this.logger = configurer.logger;
-		this.prognosticator = new GraphPropertyPrognosticator(app);
+		this.prognosticator = configurer.prognosticator;
 		this.mLogger = configurer.mLogger;
 	}
 
@@ -79,7 +72,7 @@ private void tune() {
 			Pair<Boolean, Long> ret;
 
 			System.out.println("New tune run.............");
-			while (manager.getStatus() != AppStatus.STOPPED) {
+			while (configurer.manager.getStatus() != AppStatus.STOPPED) {
 				mLogger.bTuningRound();
 				String cfgJson = tuner.readLine();
 				logger.logSearchTime(searchTimeSW
@@ -189,7 +182,7 @@ private Configuration newCfg(int round, String cfgJson) {
 
 	private void tuningFinished() {
 		try {
-			drainer.dumpDraindataStatistics();
+			configurer.drainer.dumpDraindataStatistics();
 		} catch (IOException e) {
 			e.printStackTrace();
 		}

From 5b3683148a44efed2d81b0ac3e39c47d4d32e76e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 10:58:36 +0800
Subject: [PATCH 858/881] Unused imports removed.

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 2681324c..45c77729 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -1,10 +1,7 @@
 package edu.mit.streamjit.tuner;
 
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;

From 7864ef1379b7614c91a5e02f195a1c7499b189a4 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 17:02:20 +0800
Subject: [PATCH 859/881] BugFix: time>1 -> time>0

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index 45c77729..c1fd9ae0 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -96,7 +96,7 @@ private void tune() {
 				mLogger.eReconfigure();
 				if (ret.first) {
 					long time = ret.second;
-					currentBestTime = (time > 1 && currentBestTime > time)
+					currentBestTime = (time > 0 && currentBestTime > time)
 							? time : currentBestTime;
 					prognosticator.time(ret.second);
 					tuner.writeLine(new Double(ret.second).toString());

From 05f8cee82a9a0c7d0333f92b02a42c458f88b83e Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Wed, 11 Mar 2015 23:33:01 +0800
Subject: [PATCH 860/881] Minor optimization in TimeLog processing

if -> else if
---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 37df3165..c2c8db3c 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -60,7 +60,7 @@ private static Map<String, Integer> processRunTime(String appName,
 		while ((line = reader.readLine()) != null) {
 			if (line.startsWith("----------------------------"))
 				cfgPrefix = cfgString(line);
-			if (line.startsWith("Execution")) {
+			else if (line.startsWith("Execution")) {
 				String[] arr = line.split(" ");
 				String time = arr[3].trim();
 				time = time.substring(0, time.length() - 2);
@@ -119,7 +119,7 @@ private static Map<String, Integer> process(BufferedReader reader,
 		while ((line = reader.readLine()) != null) {
 			if (line.startsWith("----------------------------"))
 				cfgPrefix = cfgString(line);
-			if (line.startsWith(startString)) {
+			else if (line.startsWith(startString)) {
 				String[] arr = line.split(" ");
 				String time = arr[timepos].trim();
 				time = time.substring(0, time.length() - timeUnitLength);

From 62853155154fd49b1dd9945bb21a5cddbe0f7f53 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:11:09 +0800
Subject: [PATCH 861/881] GnuPlot:Yaxis's label changed changed from ms to s

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index c2c8db3c..b37b6cf2 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -246,6 +246,7 @@ private static File makePlotFile(File dir, String name, String dataFile)
 		writer.write(String
 				.format("plot \"%s\" using 1:5 with linespoints title \"Drain time\"\n",
 						dataFile));
+		writer.write("set ylabel \"Time(s)\"\n");
 		writer.write(String
 				.format("plot \"%s\" using 1:2 with linespoints title \"Tuning Round time\"\n",
 						dataFile));

From 630b65797643ef4f5d7fca8795bdb594f223f596 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:14:38 +0800
Subject: [PATCH 862/881] GnuPlot: plots Runtime before compile time.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index b37b6cf2..94b302f1 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -237,12 +237,12 @@ private static File makePlotFile(File dir, String name, String dataFile)
 		writer.write(String
 				.format("plot \"%s\" using 1:6 with linespoints title \"Current best running time\"\n",
 						dataFile));
-		writer.write(String
-				.format("plot \"%s\" using 1:3 with linespoints title \"Compile time\"\n",
-						dataFile));
 		writer.write(String.format(
 				"plot \"%s\" using 1:4 with linespoints title \"Run time\"\n",
 				dataFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:3 with linespoints title \"Compile time\"\n",
+						dataFile));
 		writer.write(String
 				.format("plot \"%s\" using 1:5 with linespoints title \"Drain time\"\n",
 						dataFile));

From 34eefa65f73715e123cb989d08f63e5fcfde2260 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:27:06 +0800
Subject: [PATCH 863/881] TimeLogProcessor: output file names hard coded.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 94b302f1..be01a419 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -23,6 +23,13 @@
  */
 public class TimeLogProcessor {
 
+	private static final String pRunTimeFile = "processedRunTime.txt";
+	private static final String pCompTimeFile = "processedCompileTime.txt";
+	private static final String pDrainTimeFile = "processedDrainTime.txt";
+	private static final String pTuneRoundTimeFile = "processedTuningRoundTime.txt";
+	private static final String pEvalTimeFile = "processedEvaluation.txt";
+	private static final String ptotalFile = "totalStats.txt";
+
 	public static void main(String[] args) throws IOException {
 		summarize("FMRadioCore");
 		summarizeHeap("FMRadioCore");
@@ -33,7 +40,7 @@ private static Map<String, Integer> processCompileTime(String appName,
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%scompileTime.txt", appName, File.separator)));
 
-		File outFile = new File(outDir, "processedCompileTime.txt");
+		File outFile = new File(outDir, pCompTimeFile);
 		FileWriter writer = new FileWriter(outFile, false);
 		Map<String, Integer> ret = process(reader, writer, "Total", true, 3);
 		reader.close();
@@ -50,7 +57,7 @@ private static Map<String, Integer> processRunTime(String appName,
 			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-		File outFile = new File(outDir, "processedRunTime.txt");
+		File outFile = new File(outDir, pRunTimeFile);
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		String cfgPrefix = "Init";
@@ -79,12 +86,11 @@ else if (line.startsWith("Execution")) {
 		writer.close();
 		return ret;
 	}
-
 	private static Map<String, Integer> processDrainTime(String appName,
 			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sdrainTime.txt", appName, File.separator)));
-		File outFile = new File(outDir, "processedDrainTime.txt");
+		File outFile = new File(outDir, pDrainTimeFile);
 		FileWriter writer = new FileWriter(outFile, false);
 		Map<String, Integer> ret = process(reader, writer, "Drain time", true,
 				3);
@@ -98,7 +104,7 @@ private static Map<String, Integer> processTuningRoundTime(String appName,
 			File outDir) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-		File outFile = new File(outDir, "processedTuningRoundTime.txt");
+		File outFile = new File(outDir, pTuneRoundTimeFile);
 		FileWriter writer = new FileWriter(outFile, false);
 		Map<String, Integer> ret = process(reader, writer, "Tuning", false, 3);
 		reader.close();
@@ -185,9 +191,8 @@ public static void summarize(String appName) throws IOException {
 		Map<String, Integer> drainTime = processDrainTime(appName, summaryDir);
 		Map<String, Integer> tuningRoundTime = processTuningRoundTime(appName,
 				summaryDir);
-		String dataFile = "totalStats.txt";
 
-		File outfile = new File(summaryDir, dataFile);
+		File outfile = new File(summaryDir, ptotalFile);
 		FileWriter writer = new FileWriter(outfile, false);
 		FileWriter verify = new FileWriter(String.format("%s%sverify.txt",
 				appName, File.separator));
@@ -213,7 +218,7 @@ public static void summarize(String appName) throws IOException {
 		verify.close();
 		writer.close();
 
-		File f = makePlotFile(summaryDir, appName, dataFile);
+		File f = makePlotFile(summaryDir, appName, ptotalFile);
 		plot(summaryDir, f);
 	}
 
@@ -359,7 +364,7 @@ public static void processEvaltxt(String appName) throws IOException {
 		Utils.createDir(summaryDir.getPath());
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%sevaluation.txt", appName, File.separator)));
-		File outFile = new File(summaryDir, "processedEvaluation.txt");
+		File outFile = new File(summaryDir, pEvalTimeFile);
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
 		int i = 0;

From ec8ebefa40f0d287c0aa6973b71c4fd4bd55261f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:34:35 +0800
Subject: [PATCH 864/881] Renamed:makePlotFile()->createTotalStatsPlotFile()

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index be01a419..ac1aede0 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -218,12 +218,12 @@ public static void summarize(String appName) throws IOException {
 		verify.close();
 		writer.close();
 
-		File f = makePlotFile(summaryDir, appName, ptotalFile);
+		File f = createTotalStatsPlotFile(summaryDir, appName, ptotalFile);
 		plot(summaryDir, f);
 	}
 
-	private static File makePlotFile(File dir, String name, String dataFile)
-			throws IOException {
+	private static File createTotalStatsPlotFile(File dir, String name,
+			String dataFile) throws IOException {
 		boolean pdf = true;
 		File plotfile = new File(dir, "plot.plt");
 		FileWriter writer = new FileWriter(plotfile, false);

From 7ae56104cf0bdab3164fa2dc00d004f6aea01095 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:36:33 +0800
Subject: [PATCH 865/881] outputfile name changed (plot.plt->totalstats.plt)

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index ac1aede0..8580ab84 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -225,7 +225,7 @@ public static void summarize(String appName) throws IOException {
 	private static File createTotalStatsPlotFile(File dir, String name,
 			String dataFile) throws IOException {
 		boolean pdf = true;
-		File plotfile = new File(dir, "plot.plt");
+		File plotfile = new File(dir, "totalStats.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
 		if (pdf) {
 			writer.write("set terminal pdf enhanced color\n");

From 4ab25266e266b0a4417a09d023c3f5e16cf1c6cb Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 00:44:33 +0800
Subject: [PATCH 866/881] Unnecessary method param removed.

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 8580ab84..9ed6c2d9 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -218,12 +218,12 @@ public static void summarize(String appName) throws IOException {
 		verify.close();
 		writer.close();
 
-		File f = createTotalStatsPlotFile(summaryDir, appName, ptotalFile);
+		File f = createTotalStatsPlotFile(summaryDir, appName);
 		plot(summaryDir, f);
 	}
 
-	private static File createTotalStatsPlotFile(File dir, String name,
-			String dataFile) throws IOException {
+	private static File createTotalStatsPlotFile(File dir, String name)
+			throws IOException {
 		boolean pdf = true;
 		File plotfile = new File(dir, "totalStats.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
@@ -241,20 +241,20 @@ private static File createTotalStatsPlotFile(File dir, String name,
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
 				.format("plot \"%s\" using 1:6 with linespoints title \"Current best running time\"\n",
-						dataFile));
+						ptotalFile));
 		writer.write(String.format(
 				"plot \"%s\" using 1:4 with linespoints title \"Run time\"\n",
-				dataFile));
+				ptotalFile));
 		writer.write(String
 				.format("plot \"%s\" using 1:3 with linespoints title \"Compile time\"\n",
-						dataFile));
+						ptotalFile));
 		writer.write(String
 				.format("plot \"%s\" using 1:5 with linespoints title \"Drain time\"\n",
-						dataFile));
+						ptotalFile));
 		writer.write("set ylabel \"Time(s)\"\n");
 		writer.write(String
 				.format("plot \"%s\" using 1:2 with linespoints title \"Tuning Round time\"\n",
-						dataFile));
+						ptotalFile));
 		writer.close();
 		return plotfile;
 	}

From a5bb76ac281c251b6665375209488c7d24670209 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 01:14:11 +0800
Subject: [PATCH 867/881] createProcessedPlotFile() added.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 9ed6c2d9..66173bcb 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -220,8 +220,13 @@ public static void summarize(String appName) throws IOException {
 
 		File f = createTotalStatsPlotFile(summaryDir, appName);
 		plot(summaryDir, f);
+		f = createProcessedPlotFile(summaryDir, appName);
+		plot(summaryDir, f);
 	}
 
+	/**
+	 * Creates plot file for {@link #ptotalFile}.
+	 */
 	private static File createTotalStatsPlotFile(File dir, String name)
 			throws IOException {
 		boolean pdf = true;
@@ -259,6 +264,48 @@ private static File createTotalStatsPlotFile(File dir, String name)
 		return plotfile;
 	}
 
+	/**
+	 * Creates a plot file that uses data from all processed files (
+	 * {@link #pRunTimeFile}, {@link #pCompTimeFile}, {@link #pDrainTimeFile}
+	 * and {@link #pTuneRoundTimeFile}).
+	 */
+	private static File createProcessedPlotFile(File dir, String name)
+			throws IOException {
+		boolean pdf = true;
+		File plotfile = new File(dir, "processed.plt");
+		FileWriter writer = new FileWriter(plotfile, false);
+		if (pdf) {
+			writer.write("set terminal pdf enhanced color\n");
+			writer.write(String.format("set output \"%sP.pdf\"\n", name));
+		} else {
+			writer.write("set terminal postscript eps enhanced color\n");
+			writer.write(String.format("set output \"%s.eps\"\n", name));
+		}
+		writer.write("set ylabel \"Time(ms)\"\n");
+		writer.write("set xlabel \"Tuning Rounds\"\n");
+		writer.write(String.format("set title \"%s\"\n", name));
+		writer.write("set grid\n");
+		writer.write("#set yrange [0:*]\n");
+		writer.write(String
+				.format("plot \"%s\" using 1:4 with linespoints title \"Current best running time\"\n",
+						pRunTimeFile));
+		writer.write(String.format(
+				"plot \"%s\" using 1:3 with linespoints title \"Run time\"\n",
+				pRunTimeFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:3 with linespoints title \"Compile time\"\n",
+						pCompTimeFile));
+		writer.write(String
+				.format("plot \"%s\" using 1:3 with linespoints title \"Drain time\"\n",
+						pDrainTimeFile));
+		writer.write("set ylabel \"Time(s)\"\n");
+		writer.write(String
+				.format("plot \"%s\" using 1:3 with linespoints title \"Tuning Round time\"\n",
+						pTuneRoundTimeFile));
+		writer.close();
+		return plotfile;
+	}
+
 	private static File makeHeapPlotFile(File dir, String name,
 			String dataFile1, String dataFile2) throws IOException {
 		boolean pdf = true;

From c324dab17872ba9be9db97ae0693de2a10d4a26b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 09:17:18 +0800
Subject: [PATCH 868/881] Renamed: name->appName

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 66173bcb..d15222d7 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -227,21 +227,21 @@ public static void summarize(String appName) throws IOException {
 	/**
 	 * Creates plot file for {@link #ptotalFile}.
 	 */
-	private static File createTotalStatsPlotFile(File dir, String name)
+	private static File createTotalStatsPlotFile(File dir, String appName)
 			throws IOException {
 		boolean pdf = true;
 		File plotfile = new File(dir, "totalStats.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
 		if (pdf) {
 			writer.write("set terminal pdf enhanced color\n");
-			writer.write(String.format("set output \"%s.pdf\"\n", name));
+			writer.write(String.format("set output \"%s.pdf\"\n", appName));
 		} else {
 			writer.write("set terminal postscript eps enhanced color\n");
-			writer.write(String.format("set output \"%s.eps\"\n", name));
+			writer.write(String.format("set output \"%s.eps\"\n", appName));
 		}
 		writer.write("set ylabel \"Time(ms)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
-		writer.write(String.format("set title \"%s\"\n", name));
+		writer.write(String.format("set title \"%s\"\n", appName));
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
@@ -269,21 +269,21 @@ private static File createTotalStatsPlotFile(File dir, String name)
 	 * {@link #pRunTimeFile}, {@link #pCompTimeFile}, {@link #pDrainTimeFile}
 	 * and {@link #pTuneRoundTimeFile}).
 	 */
-	private static File createProcessedPlotFile(File dir, String name)
+	private static File createProcessedPlotFile(File dir, String appName)
 			throws IOException {
 		boolean pdf = true;
 		File plotfile = new File(dir, "processed.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
 		if (pdf) {
 			writer.write("set terminal pdf enhanced color\n");
-			writer.write(String.format("set output \"%sP.pdf\"\n", name));
+			writer.write(String.format("set output \"%sP.pdf\"\n", appName));
 		} else {
 			writer.write("set terminal postscript eps enhanced color\n");
-			writer.write(String.format("set output \"%s.eps\"\n", name));
+			writer.write(String.format("set output \"%s.eps\"\n", appName));
 		}
 		writer.write("set ylabel \"Time(ms)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
-		writer.write(String.format("set title \"%s\"\n", name));
+		writer.write(String.format("set title \"%s\"\n", appName));
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String

From c99529fdc57d8b74d4fa382f4ec95fca6cf2af93 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 09:28:56 +0800
Subject: [PATCH 869/881] Gnuplot: Uses benchmark name for graph title.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 34 +++++++++++++++----
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index d15222d7..4e1b47b8 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -2,6 +2,7 @@
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -11,6 +12,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
@@ -229,19 +231,20 @@ public static void summarize(String appName) throws IOException {
 	 */
 	private static File createTotalStatsPlotFile(File dir, String appName)
 			throws IOException {
+		String title = getTitle(appName);
 		boolean pdf = true;
 		File plotfile = new File(dir, "totalStats.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
 		if (pdf) {
 			writer.write("set terminal pdf enhanced color\n");
-			writer.write(String.format("set output \"%s.pdf\"\n", appName));
+			writer.write(String.format("set output \"%s.pdf\"\n", title));
 		} else {
 			writer.write("set terminal postscript eps enhanced color\n");
-			writer.write(String.format("set output \"%s.eps\"\n", appName));
+			writer.write(String.format("set output \"%s.eps\"\n", title));
 		}
 		writer.write("set ylabel \"Time(ms)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
-		writer.write(String.format("set title \"%s\"\n", appName));
+		writer.write(String.format("set title \"%s\"\n", title));
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
@@ -271,19 +274,20 @@ private static File createTotalStatsPlotFile(File dir, String appName)
 	 */
 	private static File createProcessedPlotFile(File dir, String appName)
 			throws IOException {
+		String title = getTitle(appName);
 		boolean pdf = true;
 		File plotfile = new File(dir, "processed.plt");
 		FileWriter writer = new FileWriter(plotfile, false);
 		if (pdf) {
 			writer.write("set terminal pdf enhanced color\n");
-			writer.write(String.format("set output \"%sP.pdf\"\n", appName));
+			writer.write(String.format("set output \"%sP.pdf\"\n", title));
 		} else {
 			writer.write("set terminal postscript eps enhanced color\n");
-			writer.write(String.format("set output \"%s.eps\"\n", appName));
+			writer.write(String.format("set output \"%s.eps\"\n", title));
 		}
 		writer.write("set ylabel \"Time(ms)\"\n");
 		writer.write("set xlabel \"Tuning Rounds\"\n");
-		writer.write(String.format("set title \"%s\"\n", appName));
+		writer.write(String.format("set title \"%s\"\n", title));
 		writer.write("set grid\n");
 		writer.write("#set yrange [0:*]\n");
 		writer.write(String
@@ -475,4 +479,22 @@ public static void processVerifycaionRun(String appName) throws IOException {
 		processVerifycaionRunTime(appName, summaryDir,
 				"final_NestedSplitJoinCore.cfg");
 	}
+
+	private static String getTitle(String appName) {
+		String benchmarkName = getBenchmarkName(appName);
+		return benchmarkName == null ? appName : benchmarkName;
+	}
+
+	private static String getBenchmarkName(String appName) {
+		Properties prop = new Properties();
+		InputStream input = null;
+		try {
+			input = new FileInputStream(String.format("%s%sREADME.txt",
+					appName, File.separator));
+			prop.load(input);
+		} catch (IOException ex) {
+			System.err.println("Failed to load README.txt");
+		}
+		return prop.getProperty("benchmarkName");
+	}
 }

From 9757197a582c0d82973432c9d229357955fe8f11 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 20:20:17 +0800
Subject: [PATCH 870/881] Verifier.cfgPrefixes() has been made public static

So that TimeLogProcessor can use this mathod to read the verify.txt.
---
 src/edu/mit/streamjit/tuner/Verifier.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index 973b5d11..7553b804 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -28,7 +28,7 @@ public Verifier(Reconfigurer configurer) {
 	}
 
 	public void verify() {
-		verifyTuningTimes(cfgPrefixes());
+		verifyTuningTimes(cfgPrefixes(appName));
 	}
 
 	/**
@@ -100,7 +100,7 @@ private FileWriter writer() throws IOException {
 		return writer;
 	}
 
-	private Map<String, Integer> cfgPrefixes() {
+	public static Map<String, Integer> cfgPrefixes(String appName) {
 		Map<String, Integer> cfgPrefixes = new HashMap<>();
 		cfgPrefixes.put("final", 0);
 		cfgPrefixes.put("hand", 0);
@@ -123,7 +123,7 @@ private Map<String, Integer> cfgPrefixes() {
 	/**
 	 * Processes the line that is generated by {@link TimeLogProcessor}.
 	 */
-	private void process1(String line, Map<String, Integer> cfgPrefixes) {
+	private static void process1(String line, Map<String, Integer> cfgPrefixes) {
 		String[] arr = line.split("=");
 		String cfgPrefix = arr[0].trim();
 		int expectedTime = 0;
@@ -139,7 +139,7 @@ private void process1(String line, Map<String, Integer> cfgPrefixes) {
 	/**
 	 * Processes manually entered lines in the verify.txt
 	 */
-	private void process2(String line, Map<String, Integer> cfgPrefixes) {
+	private static void process2(String line, Map<String, Integer> cfgPrefixes) {
 		String[] arr = line.split(",");
 		for (String s : arr) {
 			cfgPrefixes.put(s.trim(), 0);

From 3a3028ad97c3eb70cff94d904ca711147ef5e0c3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 20:33:14 +0800
Subject: [PATCH 871/881] cfgName removed from Verifier.evaluateConfig()

evaluateConfig() uses cfgPrefix for logging purpose instead of cfgName.
---
 src/edu/mit/streamjit/tuner/Verifier.java | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index 7553b804..a6ce5109 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -60,7 +60,7 @@ private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 					writer.write("----------------------------------------\n");
 					writer.write(String.format("Configuration name = %s\n",
 							cfgName));
-					List<Long> runningTimes = evaluateConfig(cfg, cfgName);
+					List<Long> runningTimes = evaluateConfig(cfg);
 					processRunningTimes(runningTimes, expectedRunningTime,
 							writer);
 				}
@@ -151,17 +151,16 @@ private static void process2(String line, Map<String, Integer> cfgPrefixes) {
 	 * 
 	 * @param cfg
 	 *            configuration that needs to be evaluated
-	 * @param cfgName
-	 *            name of the configuration. This is just for logging purpose.
 	 */
-	private List<Long> evaluateConfig(Configuration cfg, String cfgName) {
-		System.out.println("Evaluating " + cfgName);
+	private List<Long> evaluateConfig(Configuration cfg) {
+		String cfgPrefix = ConfigurationUtils.getConfigPrefix(cfg);
+		System.out.println("Evaluating " + cfgPrefix);
 		int count = Options.evaluationCount;
 		List<Long> runningTime = new ArrayList<>(count);
 		Pair<Boolean, Long> ret;
 		if (cfg != null) {
 			for (int i = 0; i < count; i++) {
-				configurer.logger.newConfiguration(cfgName);
+				configurer.logger.newConfiguration(cfgPrefix);
 				ret = configurer.reconfigure(cfg, 0);
 				if (ret.first) {
 					configurer.prognosticator.time(ret.second);

From a94041c50c52bf54da6abb7cbf96a8488ede7a6a Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 20:36:47 +0800
Subject: [PATCH 872/881] TimeLogProcessor processes verification run

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 4e1b47b8..99c64f66 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -16,6 +16,7 @@
 
 import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.impl.distributed.common.Utils;
+import edu.mit.streamjit.tuner.Verifier;
 
 /**
  * Processes the Distributed StreamJit's time log files and generate summary.
@@ -473,11 +474,17 @@ private static void processVerifycaionRunTime(String appName, File outDir,
 	}
 
 	public static void processVerifycaionRun(String appName) throws IOException {
+		processVerifycaionRun(appName, Verifier.cfgPrefixes(appName));
+	}
+
+	public static void processVerifycaionRun(String appName,
+			Map<String, Integer> cfgPrefixes) throws IOException {
 		File summaryDir = new File(String.format("%s%ssummary", appName,
 				File.separator));
 		Utils.createDir(summaryDir.getPath());
-		processVerifycaionRunTime(appName, summaryDir,
-				"final_NestedSplitJoinCore.cfg");
+		for (String cfgPrefix : cfgPrefixes.keySet()) {
+			processVerifycaionRunTime(appName, summaryDir, cfgPrefix);
+		}
 	}
 
 	private static String getTitle(String appName) {

From e69997641c80727316f15388845921643aa149c5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 20:41:51 +0800
Subject: [PATCH 873/881] Verifier.cfgPrefixes:Stopped adding 'hand'&'final'

Verifier.cfgPrefixes was adding 'hand' and 'final' cfg prefixes always.
This has been stopped.
---
 src/edu/mit/streamjit/tuner/Verifier.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index a6ce5109..77e92341 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -102,8 +102,6 @@ private FileWriter writer() throws IOException {
 
 	public static Map<String, Integer> cfgPrefixes(String appName) {
 		Map<String, Integer> cfgPrefixes = new HashMap<>();
-		cfgPrefixes.put("final", 0);
-		cfgPrefixes.put("hand", 0);
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(
 					String.format("%s%sverify.txt", appName, File.separator)));

From ae13d239c09902dddda4687c3a8caa0bb26bb96b Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 21:10:28 +0800
Subject: [PATCH 874/881] Generates graphs for verification run.

---
 .../mit/streamjit/util/TimeLogProcessor.java  | 37 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 99c64f66..8ea90aee 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -444,7 +444,7 @@ private static void processVerifycaionRunTime(String appName, File outDir,
 			String cfgName) throws IOException {
 		BufferedReader reader = new BufferedReader(new FileReader(
 				String.format("%s%srunTime.txt", appName, File.separator)));
-		File outFile = new File(outDir, String.format("processed%s.txt",
+		File outFile = new File(outDir, String.format("verification%s.txt",
 				cfgName));
 		FileWriter writer = new FileWriter(outFile, false);
 		String line;
@@ -484,6 +484,8 @@ public static void processVerifycaionRun(String appName,
 		Utils.createDir(summaryDir.getPath());
 		for (String cfgPrefix : cfgPrefixes.keySet()) {
 			processVerifycaionRunTime(appName, summaryDir, cfgPrefix);
+			File f = createVerificationPlotFile(summaryDir, appName, cfgPrefix);
+			plot(summaryDir, f);
 		}
 	}
 
@@ -504,4 +506,37 @@ private static String getBenchmarkName(String appName) {
 		}
 		return prop.getProperty("benchmarkName");
 	}
+
+	/**
+	 * Creates a plot file that uses data from all processed files (
+	 * {@link #pRunTimeFile}, {@link #pCompTimeFile}, {@link #pDrainTimeFile}
+	 * and {@link #pTuneRoundTimeFile}).
+	 */
+	private static File createVerificationPlotFile(File dir, String appName,
+			String cfgPrefix) throws IOException {
+		String title = getTitle(appName);
+		title = String.format("%s-cfg%s", title, cfgPrefix);
+		boolean pdf = true;
+		String dataFile = String.format("verification%s.txt", cfgPrefix);
+		File plotfile = new File(dir, String.format("verification%s.plt",
+				cfgPrefix));
+		FileWriter writer = new FileWriter(plotfile, false);
+		if (pdf) {
+			writer.write("set terminal pdf enhanced color\n");
+			writer.write(String.format("set output \"%s.pdf\"\n", title));
+		} else {
+			writer.write("set terminal postscript eps enhanced color\n");
+			writer.write(String.format("set output \"%s.eps\"\n", title));
+		}
+		writer.write("set ylabel \"Time(ms)\"\n");
+		writer.write("set xlabel \"Tuning Rounds\"\n");
+		writer.write(String.format("set title \"%s\"\n", title));
+		writer.write("set grid\n");
+		writer.write("#set yrange [0:*]\n");
+		writer.write(String.format(
+				"plot \"%s\" using 1:3 with linespoints title \"Run time\"\n",
+				dataFile));
+		writer.close();
+		return plotfile;
+	}
 }

From 708fceea9d10bb7f35f7bb6c55ac83f09bd3dca2 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 21:16:15 +0800
Subject: [PATCH 875/881] Verifier generates graphs after the verification.

---
 src/edu/mit/streamjit/tuner/Verifier.java | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index 77e92341..6496deed 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -28,7 +28,17 @@ public Verifier(Reconfigurer configurer) {
 	}
 
 	public void verify() {
-		verifyTuningTimes(cfgPrefixes(appName));
+		Map<String, Integer> cfgPrefixes = cfgPrefixes(appName);
+		verifyTuningTimes(cfgPrefixes);
+	}
+
+	private void generateGraphs(Map<String, Integer> cfgPrefixes) {
+		if (Options.verificationCount > 50)
+			try {
+				TimeLogProcessor.processVerifycaionRun(appName, cfgPrefixes);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 	}
 
 	/**

From 902046e727f13d2cd2e34d884f89248ad9550d7f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 21:43:09 +0800
Subject: [PATCH 876/881] Verifier calls TimeLogProcessor.summarize()

---
 src/edu/mit/streamjit/tuner/Verifier.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index 6496deed..b55d165e 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -33,9 +33,10 @@ public void verify() {
 	}
 
 	private void generateGraphs(Map<String, Integer> cfgPrefixes) {
-		if (Options.verificationCount > 50)
+		if (Options.verificationCount > 50 || Options.evaluationCount > 50)
 			try {
 				TimeLogProcessor.processVerifycaionRun(appName, cfgPrefixes);
+				TimeLogProcessor.summarize(appName);
 			} catch (IOException e) {
 				e.printStackTrace();
 			}

From dfbeec2cd92945efcf27ffddc6282874116997b3 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 21:46:20 +0800
Subject: [PATCH 877/881] TimeLogProcessor appends to verify.txt

---
 src/edu/mit/streamjit/util/TimeLogProcessor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/edu/mit/streamjit/util/TimeLogProcessor.java b/src/edu/mit/streamjit/util/TimeLogProcessor.java
index 8ea90aee..621c6218 100644
--- a/src/edu/mit/streamjit/util/TimeLogProcessor.java
+++ b/src/edu/mit/streamjit/util/TimeLogProcessor.java
@@ -198,7 +198,7 @@ public static void summarize(String appName) throws IOException {
 		File outfile = new File(summaryDir, ptotalFile);
 		FileWriter writer = new FileWriter(outfile, false);
 		FileWriter verify = new FileWriter(String.format("%s%sverify.txt",
-				appName, File.separator));
+				appName, File.separator), true);
 		int min = Integer.MAX_VALUE;
 		writer.write("cfg\tTRTime\tcomp\trun\tdrain\tmin\n");
 		for (int i = 1; i <= tuningRoundTime.size(); i++) {

From f23b9c56853e9cff59ac2b2bad40d15f5785ba66 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 22:20:31 +0800
Subject: [PATCH 878/881] Missed change.Verifier.verify calls generateGraphs

---
 src/edu/mit/streamjit/tuner/Verifier.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index b55d165e..04434119 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -30,6 +30,7 @@ public Verifier(Reconfigurer configurer) {
 	public void verify() {
 		Map<String, Integer> cfgPrefixes = cfgPrefixes(appName);
 		verifyTuningTimes(cfgPrefixes);
+		generateGraphs(cfgPrefixes);
 	}
 
 	private void generateGraphs(Map<String, Integer> cfgPrefixes) {

From eac7b910b1772f45a9a7623fd17f35a67cb288ec Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Thu, 12 Mar 2015 22:26:47 +0800
Subject: [PATCH 879/881] handleTermination() calls verifyTuningTimes()

---
 src/edu/mit/streamjit/tuner/OnlineTuner.java | 13 ++++++++++---
 src/edu/mit/streamjit/tuner/Verifier.java    |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/edu/mit/streamjit/tuner/OnlineTuner.java b/src/edu/mit/streamjit/tuner/OnlineTuner.java
index c1fd9ae0..cf4e3a57 100644
--- a/src/edu/mit/streamjit/tuner/OnlineTuner.java
+++ b/src/edu/mit/streamjit/tuner/OnlineTuner.java
@@ -2,6 +2,8 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
@@ -149,9 +151,7 @@ private void handleTermination() throws IOException {
 		ConfigurationUtils.saveConfg(finalConfg, "final", app.name);
 		Configuration finalcfg = Configuration.fromJson(finalConfg);
 		finalcfg = ConfigurationUtils.addConfigPrefix(finalcfg, "final");
-
-		new Verifier(configurer).verify();
-
+		verify();
 		if (needTermination) {
 			configurer.terminate();
 		} else {
@@ -164,6 +164,13 @@ private void handleTermination() throws IOException {
 		}
 	}
 
+	private void verify() {
+		Map<String, Integer> cfgPrefixes = new HashMap<>();
+		cfgPrefixes.put("final", 0);
+		cfgPrefixes.put("hand", 0);
+		new Verifier(configurer).verifyTuningTimes(cfgPrefixes);
+	}
+
 	private Configuration newCfg(int round, String cfgJson) {
 		String cfgPrefix = new Integer(round).toString();
 		System.out.println(String.format(
diff --git a/src/edu/mit/streamjit/tuner/Verifier.java b/src/edu/mit/streamjit/tuner/Verifier.java
index 04434119..d3128e39 100644
--- a/src/edu/mit/streamjit/tuner/Verifier.java
+++ b/src/edu/mit/streamjit/tuner/Verifier.java
@@ -52,7 +52,7 @@ private void generateGraphs(Map<String, Integer> cfgPrefixes) {
 	 * @param cfgPrefixes
 	 *            map of cfgPrefixes and expected running time.
 	 */
-	private void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
+	void verifyTuningTimes(Map<String, Integer> cfgPrefixes) {
 		try {
 			FileWriter writer = writer();
 			for (int i = 0; i < Options.verificationCount; i++) {

From 95035043e7451ca5592156ff106c6cb0febac2b5 Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Mar 2015 11:46:06 +0800
Subject: [PATCH 880/881] OT has renamed ArrayParameter->ParameterArray

In OpenTuner v0.5.0, ArrayParameter has been renamed to ParameterArray.
---
 lib/opentuner/streamjit/sjparameters.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/opentuner/streamjit/sjparameters.py b/lib/opentuner/streamjit/sjparameters.py
index 5d7471f2..3f2862cc 100644
--- a/lib/opentuner/streamjit/sjparameters.py
+++ b/lib/opentuner/streamjit/sjparameters.py
@@ -1,7 +1,7 @@
 import deps #fix sys.path
 import random
 import opentuner
-from opentuner.search.manipulator import IntegerParameter, FloatParameter, SwitchParameter, PermutationParameter, ArrayParameter
+from opentuner.search.manipulator import IntegerParameter, FloatParameter, SwitchParameter, PermutationParameter, ParameterArray
 
 class sjIntegerParameter(IntegerParameter):
 	def __init__(self, name, min, max, value, javaClass = None, **kwargs):
@@ -86,7 +86,7 @@ def json_replacement(self):
 			"universe": self.universe,
 			"class": self.javaClass}
 
-class sjCompositionParameter(ArrayParameter):
+class sjCompositionParameter(ParameterArray):
 	def __init__(self, name, values, javaClass):
 		super(sjCompositionParameter, self).__init__(name, len(values), FloatParameter, 0.0, 1.0)
 		self.values = values

From 260948203742a1fb87aac9627e7b1938eea59e4f Mon Sep 17 00:00:00 2001
From: sumanan <sumanan@csail.mit.edu>
Date: Fri, 20 Mar 2015 12:50:14 +0800
Subject: [PATCH 881/881] Use of Options removed from Compiler2.

---
 src/edu/mit/streamjit/impl/compiler2/Compiler2.java         | 6 +-----
 .../mit/streamjit/impl/compiler2/Compiler2BlobFactory.java  | 4 +---
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
index 1ec7b571..f49f5c14 100644
--- a/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
+++ b/src/edu/mit/streamjit/impl/compiler2/Compiler2.java
@@ -22,9 +22,7 @@
 package edu.mit.streamjit.impl.compiler2;
 
 import com.google.common.base.Function;
-
 import static com.google.common.base.Preconditions.checkState;
-
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.ImmutableList;
@@ -41,7 +39,6 @@
 import com.google.common.primitives.Primitives;
 import com.google.common.reflect.TypeResolver;
 import com.google.common.reflect.TypeToken;
-
 import edu.mit.streamjit.api.DuplicateSplitter;
 import edu.mit.streamjit.api.IllegalStreamGraphException;
 import edu.mit.streamjit.api.Input;
@@ -70,7 +67,6 @@
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.DrainInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.ReadInstruction;
 import edu.mit.streamjit.impl.compiler2.Compiler2BlobHost.WriteInstruction;
-import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.test.Benchmark;
 import edu.mit.streamjit.test.Benchmarker;
 import edu.mit.streamjit.test.apps.fmradio.FMRadio;
@@ -117,7 +113,7 @@ public class Compiler2 {
 	public static final RemovalStrategy REMOVAL_STRATEGY = new BitsetRemovalStrategy();
 	public static final FusionStrategy FUSION_STRATEGY = new BitsetFusionStrategy();
 	public static final UnboxingStrategy UNBOXING_STRATEGY = new BitsetUnboxingStrategy();
-	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(Options.maxNumCores);
+	public static final AllocationStrategy ALLOCATION_STRATEGY = new SubsetBiasAllocationStrategy(8);
 	public static final StorageStrategy INTERNAL_STORAGE_STRATEGY = new TuneInternalStorageStrategy();
 	public static final StorageStrategy EXTERNAL_STORAGE_STRATEGY = new TuneExternalStorageStrategy();
 	public static final SwitchingStrategy SWITCHING_STRATEGY = SwitchingStrategy.tunePerWorker();
diff --git a/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java b/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
index 655a314a..91b4e0ab 100644
--- a/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
+++ b/src/edu/mit/streamjit/impl/compiler2/Compiler2BlobFactory.java
@@ -27,9 +27,7 @@
 import edu.mit.streamjit.impl.blob.DrainData;
 import edu.mit.streamjit.impl.common.Configuration;
 import edu.mit.streamjit.impl.common.Workers;
-import edu.mit.streamjit.impl.distributed.common.Options;
 import edu.mit.streamjit.util.affinity.Affinity;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -82,7 +80,7 @@ public Configuration getDefaultConfiguration(Set<Worker<?, ?>> workers) {
 		//add it here or we won't pick it up in the default configuration.
 //		Configuration.PermutationParameter<Integer> affinity = new Configuration.PermutationParameter<>("$affinity", Integer.class, Affinity.getMaximalAffinity());
 //		builder.addParameter(affinity);
-		return builder.addParameter(new Configuration.IntParameter("multiplier", 1, Options.multiplierMaxValue, 1))
+		return builder.addParameter(new Configuration.IntParameter("multiplier", 1, Short.MAX_VALUE, 1))
 				.build();
 	}