Data

README.md

@@ -127,5 +127,5 @@ Here is a list of what will be provisioned via vagrant and its current status:
 * OpenSOC UI
 * OpenSOC Storm Topologies - DONE
 * Snort - DONE
-* Pycaoa - DONE
+* Pycapa - DONE
 * Bro - DONE

Vagrantfile

@@ -90,7 +90,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                     s.path = "scripts/setup-elasticsearch.sh"
                     s.args = "-i 10.0.0.10#{i}"
                 end
-                # reload supervisord
             end
 
             #After everything is provisioned, start Supervisor

fabfile.py

@@ -10,7 +10,7 @@
 temp_ssh_config = '.ssh_config'
 
 def vagrant():
-    '''sets up fabric environment to work with vagrant VMs'''
+    '''Sets up fabric environment to work with vagrant VMs'''
     with open(temp_ssh_config, 'w') as f:
         f.write(local('vagrant ssh-config', capture=True))
 
@@ -36,17 +36,31 @@ def supervisorctl_stop(process):
     '''Stop a process managed by supervisor'''
     sudo('supervisorctl stop {0}'.format(process))
 
+def supervisorctl_startall():
+    sudo('pgrep supervisord || start supervisor', warn_only=True)
+    sudo('supervisorctl start all')
+
+def startall():
+    '''Ensure that all services are up and running'''
+    for x in range(total_nodes,0,-1):
+        execute(supervisorctl_startall, host='node{0}'.format(x))
+
+def supervisorctl_stopall():
+    sudo('supervisorctl stop all')
+
+def restartall():
+    '''Restart all services'''
+    for x in range(total_nodes,0,-1):
+        execute(supervisorctl_stopall, host='node{0}'.format(x))
+    for x in range(total_nodes,0,-1):
+        execute(supervisorctl_startall, host='node{0}'.format(x))
+
 
 def postsetup():
     '''Perform post vagrant up tasks on cluster'''
+
     execute(format_namenode)
-    execute(supervisorctl_start, 'namenode', host='node1')
-    execute(supervisorctl_start, 'resourcemanager', host='node1')
-    execute(supervisorctl_start, 'master', host='node1')
-    for x in range(2,total_nodes+1):
-        execute(supervisorctl_start, 'datanode', host='node{0}'.format(x))
-        execute(supervisorctl_start, 'nodemanager', host='node{0}'.format(x))
-        execute(supervisorctl_start, 'regionserver', host='node{0}'.format(x))
+    execute(startall)
 
     execute(init_ip_whitelist,host='node1')
 
@@ -61,8 +75,10 @@ def supervisorctl_status():
     sudo('supervisorctl status')
 
 def status():
+    '''Check the status of all services'''
     execute(supervisorctl_status, hosts=['node{0}'.format(x) for x in range(1,total_nodes+1)])
 
+@hosts('node1')
 def init_ip_whitelist():
     run('/opt/hbase/bin/hbase shell /vagrant/resources/opensoc/hbase_ip_whitelist.rb')
 
@@ -137,7 +153,7 @@ def start_topology(topology, repo=None, local_mode=False, config_path='/vagrant/
             ))
 
 def restart_storm():
-    ''' restarts storm workers and nimbus'''
+    '''Restarts storm workers and nimbus'''
 
     execute(supervisorctl_stop, 'storm-nimbus', host='node1')
     execute(supervisorctl_stop, 'storm-supervisor', hosts=[ 'node{0}'.format(x) for x in range(2, total_nodes+1)])

resources/zookeeper/supervisor-zookeeper.conf

@@ -1,6 +1,7 @@
 [program:zookeeper]
 command=/opt/zookeeper/bin/zkServer.sh start-foreground
 directory=/opt/zookeeper
+autostart=false
 stdout_logfile=/var/log/zookeeper/stdout.log
 stderr_logfile=/var/log/zookeeper/stderr.log
 redirect_stderr=true

scripts/closest-mirror.py

@@ -14,9 +14,12 @@
 
 response = json.loads(urllib2.urlopen(closer_url).read())
 
-
 path = response['path_info']
 
+#added this 01/09/15
+path = format(args.project)
+
+
 if args.version:
     path = os.path.join(path, args.version)
 

scripts/common.sh

@@ -3,19 +3,23 @@
 JRE_RPM=jre-7u79-linux-x64.rpm
 HADOOP_VERSION=hadoop-2.6.0
 ZOOKEEPER_VERSION=zookeeper-3.4.6
-KAFKA_SCALA_VERSION=2.9.2
-KAFKA_VERSION_NUM=0.8.1.1
+KAFKA_SCALA_VERSION=2.10
+KAFKA_VERSION_NUM=0.8.2.1
 KAFKA_VERSION="kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION_NUM}"
 STORM_VERSION=apache-storm-0.9.4
-HBASE_VERSION_NUM=0.98.12.1
+HBASE_VERSION_NUM=0.98.14
 HBASE_VERSION=hbase-"${HBASE_VERSION_NUM}-hadoop2"
-HIVE_VERSION=hive-1.2.0
-ES_VERSION=1.5.2
+HIVE_VERSION=hive-1.2.1
+ES_VERSION=1.7.1
 FLUME_VERSION=1.6.0
 
 # So we dont need to pass in i to the scripts
 NODE_NUMBER=`hostname | tr -d node`
 
+if  [ ! -d  "/vagrant/resources/tmp" ]; then
+        echo "Creating Temporal Folder"
+        mkdir "/vagrant/resources/tmp"
+fi
 
 function downloadFile {
 
@@ -62,7 +66,7 @@ function safeSymLink {
     symlink=$2
 
     if [ -e $symlink ]; then
-        echo "${symlink} exists. Deleteing."
+        echo "${symlink} exists. Deleting."
         rm $symlink
     fi
 
@@ -75,4 +79,4 @@ function commentLine {
 
     echo "Commenting out '${line}' from ${file}"
     sed -i "s/^${line}/# ${line}/" $file
-}
+}

scripts/data/setup-pycapa.sh

@@ -3,7 +3,7 @@
 source "/vagrant/scripts/common.sh"
 
 function installPycapa {
-
+    yum install -y git
     git clone https://github.com/OpenSOC/pycapa /opt/pycapa
     cd /opt/pycapa
     pip install -r requirements.txt

scripts/data/setup-snort.sh

@@ -2,9 +2,9 @@
 
 source "/vagrant/scripts/common.sh"
 
-DAQ_VER=2.0.5-1
-SNORT_VER=2.9.7.3-1
-RULES_TARBALL=snortrules-snapshot-2973.tar.gz
+DAQ_VER=2.0.6-1
+SNORT_VER=2.9.7.5-1
+RULES_TARBALL=snortrules-snapshot-2975.tar.gz
 
 function installDeps {
     echo "installing dependencies"
@@ -19,15 +19,14 @@ function installSnort {
     yum localinstall -y "/root/rpmbuild/RPMS/x86_64/daq-${DAQ_VER}.x86_64.rpm"
 
     downloadFile "https://www.snort.org/downloads/snort/snort-${SNORT_VER}.src.rpm" "snort-${SNORT_VER}.src.rpm"
-    rpmbuild --rebuild $TARBALL
+    rpmbuild -D 'debug_package %{nil}' --rebuild $TARBALL
     yum localinstall -y "/root/rpmbuild/RPMS/x86_64/snort-${SNORT_VER}.x86_64.rpm"
 
     mkdir -p /usr/local/lib/snort_dynamicrules
 
 }
 
 function configureSnort {
-
     echo "installing local rules"
     tar -xzf "/vagrant/resources/data/${RULES_TARBALL}" -C /etc/snort
 

scripts/setup-elasticsearch.sh

@@ -2,6 +2,9 @@
 
 source "/vagrant/scripts/common.sh"
 
+ELASTIC_PATH="/opt/elasticsearch"
+
+
 while getopts ci: option; do
     case $option in
         c) ES_CLIENT=yes;;
@@ -11,35 +14,41 @@ done
 
 function installElasticsearch {
 
-    downloadFile "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz" "elasticsearch-${ES_VERSION}.tar.gz"
+    downloadFile "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz" "elastics$
 
     tar -oxf $TARBALL -C /opt
-    safeSymLink "/opt/elasticsearch-${ES_VERSION}" /opt/elasticsearch
+    safeSymLink "/opt/elasticsearch-${ES_VERSION}" $ELASTIC_PATH 
 
     mkdir -p /var/lib/elasticsearch
     mkdir -p /var/log/elasticsearch
-    mkdir -p /opt/elasticsearch/plugins
+    mkdir -p $ELASTIC_PATH/plugins
 }
 
 function configureElasticsearch {
 
     hostname=`hostname -f`
     if [ -z "${ES_CLIENT}" ]; then
         echo "Configuring elasticsearch as a normal node"
-        sed "s/__HOSTNAME__/${hostname}/" /vagrant/resources/elasticsearch/elasticsearch.yml | sed "s/__IP_ADDR__/${IP_ADDR}/" > /opt/elasticsearch/config/elasticsearch.yml
+        sed "s/__HOSTNAME__/${hostname}/" /vagrant/resources/elasticsearch/elasticsearch.yml | sed "s/__IP_ADDR__/${IP_$
     else 
         echo "Configuring elasticsearch as a client"
-        sed "s/__HOSTNAME__/${hostname}/" /vagrant/resources/elasticsearch/elasticsearch-client.yml | sed "s/__IP_ADDR__/${IP_ADDR}/" > /opt/elasticsearch/config/elasticsearch.yml
+        sed "s/__HOSTNAME__/${hostname}/" /vagrant/resources/elasticsearch/elasticsearch-client.yml | sed "s/__IP_ADDR_$
     fi
 
-    if [ ! -e /opt/elasticsearch/plugins/kopf ]; then
+    if [ ! -e $ELASTIC_PATH/plugins/kopf ]; then
         echo "Installing kopf plugin"
-        /opt/elasticsearch/bin/plugin --install lmenezes/elasticsearch-kopf/1.5.3
+        $ELASTIC_PATH/bin/plugin --install lmenezes/elasticsearch-kopf/1.5.6
     fi
 
     cp /vagrant/resources/elasticsearch/supervisor-elasticsearch.conf /etc/supervisor.d/elasticsearch.conf
 
+    echo "export PATH=\$PATH:$ELASTIC_PATH/bin/" >> /home/vagrant/.bash_profile
+
+    source /home/vagrant/.bash_profile  
+
 }
 echo "Setting up Elasticsearch"
 installElasticsearch
-configureElasticsearch
+configureElasticsearch
+
+

scripts/setup-hadoop.sh

@@ -2,6 +2,8 @@
 
 source "/vagrant/scripts/common.sh"
 
+HADOOP_PATH=/opt/hadoop
+
 while getopts r:t: option; do
     case $option in
         t) TOTAL_NODES=$OPTARG;;
@@ -14,12 +16,12 @@ function installHadoop {
     downloadApacheFile hadoop/common $HADOOP_VERSION "${HADOOP_VERSION}.tar.gz"
 
     tar -oxzf $TARBALL -C /opt
-    safeSymLink "/opt/${HADOOP_VERSION}/" /opt/hadoop
+    safeSymLink "/opt/${HADOOP_VERSION}/" $HADOOP_PATH
 
     mkdir -p /var/lib/hadoop/hdfs/namenode
     mkdir -p /var/lib/hadoop/hdfs/datanode
     mkdir -p /var/log/hadoop
-    mkdir -p /opt/hadoop/logs
+    mkdir -p $HADOOP_PATH/logs
 
     # neeed for writing to HDFS
     yum install -y snappy snappy-devel
@@ -30,15 +32,21 @@ function configureHadoop {
     HADOOP_RESOURCE_DIR=/vagrant/resources/hadoop
     for file in `ls ${HADOOP_RESOURCE_DIR}/*.xml`; do
         echo "Copying ${file}"
-        cp $file /opt/hadoop/etc/hadoop
+        cp $file $HADOOP_PATH/etc/hadoop
     done
 
     echo "Setting slaves file"
+    echo "#localhost" > $HADOOP_PATH/etc/hadoop/slaves
     for i in $(seq 2 $TOTAL_NODES); do
-        echo "node${i}" >> /opt/hadoop/etc/hadoop/slaves
+        echo "node${i}" >> $HADOOP_PATH/etc/hadoop/slaves
     done
 
-    echo "export JAVA_LIBRARY_PATH=\${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native:/usr/lib64" >> /opt/hadoop/etc/hadoop/hadoop-env.sh
+    echo "export JAVA_LIBRARY_PATH=\${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native:/usr/lib64" >> $HADOOP_PATH/etc/hadoop/hadoop-env.sh
+
+    echo "export PATH=\$PATH:$HADOOP_PATH/bin/" >> /home/vagrant/.bash_profile
+
+    source /home/vagrant/.bash_profile  
+
 }
 
 function configureNameNode {
@@ -60,4 +68,4 @@ if [ "${ROLE}" == "namenode" ]; then
     configureNameNode
 elif [ "${ROLE}" == "datanode" ]; then
     configureDataNode
-fi
+fi

scripts/setup-hbase.sh

@@ -2,6 +2,8 @@
 
 source "/vagrant/scripts/common.sh"
 
+HBASE_PATH=/opt/hbase
+
 while getopts t:r: option; do
     case $option in
         t) TOTAL_NODES=$OPTARG;;
@@ -13,19 +15,26 @@ function installHbase {
     downloadApacheFile hbase $HBASE_VERSION_NUM "${HBASE_VERSION}-bin.tar.gz"
 
     tar -oxzf $TARBALL -C /opt
-    safeSymLink "/opt/${HBASE_VERSION}" /opt/hbase
+    safeSymLink "/opt/${HBASE_VERSION}" $HBASE_PATH
 
     mkdir -p /var/log/hbase
 }
 
 function configureHbase {
 
     generateZkStringNoPorts $TOTAL_NODES
-    sed "s/__ZK_QUORUM__/${ZK_STRING_NOPORTS}/" /vagrant/resources/hbase/hbase-site.xml > /opt/hbase/conf/hbase-site.xml
+    sed "s/__ZK_QUORUM__/${ZK_STRING_NOPORTS}/" /vagrant/resources/hbase/hbase-site.xml > $HBASE_PATH/conf/hbase-site.x$
     cp "/vagrant/resources/hbase/supervisor-${HBASE_ROLE}.conf" /etc/supervisor.d/hbase.conf
+
+    echo "export PATH=\$PATH:$HBASE_PATH/bin/" >> /home/vagrant/.bash_profile
+
+    source /home/vagrant/.bash_profile  
+
 }
 
 echo "Setting up HBase"
 installHbase
 configureHbase
 
+
+

scripts/setup-java.sh

@@ -1,8 +1,8 @@
 source "/vagrant/scripts/common.sh"
 
 function installJava {
-
-    rpm -q jre
+    #can be either jdk or jre
+    rpm -q ${JRE_RPM:0:3} 
     if [ $? -eq 0 ]; then
         echo "Java is already installed"
     else
@@ -13,7 +13,7 @@ function installJava {
 
 function setupEnvVars {
     echo "creating java environment variables"
-    echo export JAVA_HOME=/usr/java/default >> /etc/profile.d/java.sh
+    echo export JAVA_HOME=/usr/java/default > /etc/profile.d/java.sh
     echo export PATH=\${JAVA_HOME}/bin:\${PATH} >> /etc/profile.d/java.sh
 }