From ba0fa2fad7fa2770831181a82dabd0289d4d6ecf Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Mon, 8 Jul 2013 10:51:25 +0700 Subject: [PATCH 1/6] change config buidbot for support new version --- project_template/buildbot_config/master.cfg | 5 ++++- project_template/buildbot_config/master_svn.cfg | 5 ++++- proteus/install_buildbot_master_env.py | 4 ++-- proteus/install_buildbot_slave_env.py | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/project_template/buildbot_config/master.cfg b/project_template/buildbot_config/master.cfg index d7b2e19..7e6f9d7 100644 --- a/project_template/buildbot_config/master.cfg +++ b/project_template/buildbot_config/master.cfg @@ -52,6 +52,7 @@ c['change_source'] = [ gitpoller_develop ] from buildbot.changes import filter from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler builder_names = [ getattr(slaves_module, slave).builder_name for slave in modules ] @@ -60,7 +61,9 @@ scheduler = SingleBranchScheduler(name="develop-change" , change_filter = change_filter , treeStableTimer=30 , builderNames=builder_names) -c['schedulers'] = [ scheduler ] +force_scheduler = ForceScheduler(name="force" + , builderNames=builder_names) +c['schedulers'] = [ scheduler, force_scheduler ] ####### BUILDERS diff --git a/project_template/buildbot_config/master_svn.cfg b/project_template/buildbot_config/master_svn.cfg index 731215c..845f820 100644 --- a/project_template/buildbot_config/master_svn.cfg +++ b/project_template/buildbot_config/master_svn.cfg @@ -54,13 +54,16 @@ c['change_source'] = [ svnpoller_trunk ] # case, just kick off a 'runtests' build from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler builder_names = [ getattr(slaves_module, slave).builder_name for slave in modules ] scheduler = SingleBranchScheduler(name="trunk-change" , branch=None , treeStableTimer=30 , builderNames=builder_names) -c['schedulers'] = [ scheduler ] +force_scheduler = ForceScheduler(name="force" + , builderNames=builder_names) +c['schedulers'] = [ scheduler, force_scheduler ] ####### BUILDERS diff --git a/proteus/install_buildbot_master_env.py b/proteus/install_buildbot_master_env.py index 0089c1d..b9dc5eb 100644 --- a/proteus/install_buildbot_master_env.py +++ b/proteus/install_buildbot_master_env.py @@ -8,8 +8,8 @@ def install_buildbot_master_env(server, virtenv_path): sudo("virtualenv --no-site-packages %s" % (virtenv_path), user="www-data") with prefix("source %s/bin/activate" % (virtenv_path)): sudo("pip install SQLAlchemy==0.7.9", user="www-data") - sudo("pip install twisted==12.0.0", user="www-data") - sudo("pip install buildbot==0.8.4", user="www-data") + sudo("pip install twisted==13.0.0", user="www-data") + sudo("pip install buildbot==0.8.7p1", user="www-data") class Configure(Role): """ diff --git a/proteus/install_buildbot_slave_env.py b/proteus/install_buildbot_slave_env.py index 2f976e5..7540cf9 100644 --- a/proteus/install_buildbot_slave_env.py +++ b/proteus/install_buildbot_slave_env.py @@ -9,8 +9,8 @@ def install_buildbot_slave_env(server, virtenv_path): sudo('easy_install virtualenv') sudo("virtualenv --no-site-packages %s" % (virtenv_path), user="www-data") with prefix("source %s/bin/activate" % (virtenv_path)): - sudo("pip install twisted==12.0.0", user="www-data") - sudo("pip install buildbot-slave==0.8.6", user="www-data") + sudo("pip install twisted==13.0.0", user="www-data") + sudo("pip install buildbot-slave==0.8.7p1", user="www-data") class Configure(Role): """ From fa09fc11ffbbb6321e3e4125aef10c00a0178f8b Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Thu, 18 Jul 2013 17:57:55 +0700 Subject: [PATCH 2/6] add role for update keys --- proteus/update_authorize_key.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 proteus/update_authorize_key.py diff --git a/proteus/update_authorize_key.py b/proteus/update_authorize_key.py new file mode 100644 index 0000000..be7c142 --- /dev/null +++ b/proteus/update_authorize_key.py @@ -0,0 +1,38 @@ +from fabric.operations import sudo +from profab.role import Role + +def clean_n_complete(server): + sudo('''rm -rf /etc/cron.d/replace_key''') + sudo('''/etc/init.d/cron restart''') + sudo('''rm -rf /home/ubuntu/replace_key''') + sudo('''rm -rf /home/ubuntu/.ssh/authorized_keys_new''') + +def create_cron(server): + command = ('0-59 * * * *' + ' ubuntu cp /home/ubuntu/.ssh/authorized_keys_new' + ' /home/ubuntu/.ssh/authorized_keys') + sudo('''echo "%s" > /home/ubuntu/replace_key''' % command) + sudo('''ln -s /home/ubuntu/replace_key /etc/cron.d/''') + sudo('''chown root:root /home/ubuntu/replace_key''') + sudo('''/etc/init.d/cron restart''') + +def read_file(path_to_key): + outfile = open('%s' % path_to_key, 'r') + return outfile.read() + +def update_authorize_key(server, keys): + create_cron(server) + sudo('''echo "%s" > /home/ubuntu/.ssh/authorized_keys_new''' + % keys, user='ubuntu') + sudo('''cp /home/ubuntu/.ssh/authorized_keys_new''' + ''' /home/ubuntu/.ssh/authorized_keys''') + clean_n_complete(server) + +class Configure(Role): + def configure(self, server): + ''' + replace public key from client to the EC2 host + ''' + path_to_key = self.parameter + keys = read_file(path_to_key) + update_authorize_key(server, keys) From c419f18fc477ef5f5c50d9563299407562580aeb Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Tue, 30 Jul 2013 19:25:24 +0700 Subject: [PATCH 3/6] remove update auth script --- proteus/update_authorize_key.py | 38 --------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 proteus/update_authorize_key.py diff --git a/proteus/update_authorize_key.py b/proteus/update_authorize_key.py deleted file mode 100644 index be7c142..0000000 --- a/proteus/update_authorize_key.py +++ /dev/null @@ -1,38 +0,0 @@ -from fabric.operations import sudo -from profab.role import Role - -def clean_n_complete(server): - sudo('''rm -rf /etc/cron.d/replace_key''') - sudo('''/etc/init.d/cron restart''') - sudo('''rm -rf /home/ubuntu/replace_key''') - sudo('''rm -rf /home/ubuntu/.ssh/authorized_keys_new''') - -def create_cron(server): - command = ('0-59 * * * *' - ' ubuntu cp /home/ubuntu/.ssh/authorized_keys_new' - ' /home/ubuntu/.ssh/authorized_keys') - sudo('''echo "%s" > /home/ubuntu/replace_key''' % command) - sudo('''ln -s /home/ubuntu/replace_key /etc/cron.d/''') - sudo('''chown root:root /home/ubuntu/replace_key''') - sudo('''/etc/init.d/cron restart''') - -def read_file(path_to_key): - outfile = open('%s' % path_to_key, 'r') - return outfile.read() - -def update_authorize_key(server, keys): - create_cron(server) - sudo('''echo "%s" > /home/ubuntu/.ssh/authorized_keys_new''' - % keys, user='ubuntu') - sudo('''cp /home/ubuntu/.ssh/authorized_keys_new''' - ''' /home/ubuntu/.ssh/authorized_keys''') - clean_n_complete(server) - -class Configure(Role): - def configure(self, server): - ''' - replace public key from client to the EC2 host - ''' - path_to_key = self.parameter - keys = read_file(path_to_key) - update_authorize_key(server, keys) From 62e142b6bd50c466bf5ac11f86f800b890997421 Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Wed, 14 Aug 2013 15:55:41 +0700 Subject: [PATCH 4/6] change SQLAlchemy from 0.7.9 to 0.7.10 --- proteus/install_buildbot_master_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proteus/install_buildbot_master_env.py b/proteus/install_buildbot_master_env.py index b9dc5eb..aed7837 100644 --- a/proteus/install_buildbot_master_env.py +++ b/proteus/install_buildbot_master_env.py @@ -7,7 +7,7 @@ def install_buildbot_master_env(server, virtenv_path): sudo('easy_install virtualenv') sudo("virtualenv --no-site-packages %s" % (virtenv_path), user="www-data") with prefix("source %s/bin/activate" % (virtenv_path)): - sudo("pip install SQLAlchemy==0.7.9", user="www-data") + sudo("pip install SQLAlchemy==0.7.10", user="www-data") sudo("pip install twisted==13.0.0", user="www-data") sudo("pip install buildbot==0.8.7p1", user="www-data") From 1fc20b6a95857e6cec7749ec4a5cf9d9b689de87 Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Wed, 21 Aug 2013 16:43:31 +0700 Subject: [PATCH 5/6] change buildbot setting for support multiple project --- bin/complete_master_config | 2 +- project_template/buildbot_config/master.cfg | 30 ++++++++----------- project_template/buildbot_config/settings.py | 6 ---- .../buildbot_config/slaves/sqlite_slave.py | 2 +- .../slaves_svn/sqlite_svn_slave.py | 2 +- setup.py | 4 ++- tests/test_complete_master_config.py | 6 ++-- 7 files changed, 21 insertions(+), 31 deletions(-) delete mode 100644 project_template/buildbot_config/settings.py diff --git a/bin/complete_master_config b/bin/complete_master_config index d4a12be..7b814c5 100755 --- a/bin/complete_master_config +++ b/bin/complete_master_config @@ -6,7 +6,7 @@ if [ $# -lt 2 ]; then echo "git_url: URL to the repository" exit 1 else - master_config_file='buildbot_config/settings.py' + master_config_file='buildbot_config/settings/settings.py' ec2_host=$1 git_url=$2 diff --git a/project_template/buildbot_config/master.cfg b/project_template/buildbot_config/master.cfg index 7e6f9d7..cfea8d2 100644 --- a/project_template/buildbot_config/master.cfg +++ b/project_template/buildbot_config/master.cfg @@ -7,7 +7,7 @@ # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -from buildbot_config.settings import BRANCH, MASTER_HOST, PROJECT_NAME, REPOSITORY_URL +from buildbot_config.settings.settings import MASTER_HOST, PROJECT_NAME c = BuildmasterConfig = {} @@ -36,34 +36,28 @@ c['slavePortnum'] = 9989 # the 'change_source' setting tells the buildmaster how it should find out # about source code changes. Here we point to the buildbot clone of pyflakes. -from buildbot.changes.gitpoller import GitPoller -gitpoller_develop = GitPoller(REPOSITORY_URL - , project=PROJECT_NAME - , branch=BRANCH - , pollinterval=30 - ) -c['change_source'] = [ gitpoller_develop ] +source_module = __import__('buildbot_config.schedulers', fromlist=['buildbot_config', 'schedulers']) +src_modules = list_modules(source_module) + +load_modules(source_module, src_modules) + +c['change_source'] = [ getattr(source_module, source).gitpoller for source in src_modules ] ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build -from buildbot.changes import filter -from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler builder_names = [ getattr(slaves_module, slave).builder_name for slave in modules ] -change_filter = filter.ChangeFilter(project=PROJECT_NAME, branch=BRANCH) -scheduler = SingleBranchScheduler(name="develop-change" - , change_filter = change_filter - , treeStableTimer=30 - , builderNames=builder_names) -force_scheduler = ForceScheduler(name="force" - , builderNames=builder_names) -c['schedulers'] = [ scheduler, force_scheduler ] +scheduler = [ getattr(source_module, schedule).scheduler for schedule in src_modules ] +scheduler.append(ForceScheduler(name="force" + , builderNames=builder_names)) + +c['schedulers'] = scheduler ####### BUILDERS diff --git a/project_template/buildbot_config/settings.py b/project_template/buildbot_config/settings.py deleted file mode 100644 index dc4afb1..0000000 --- a/project_template/buildbot_config/settings.py +++ /dev/null @@ -1,6 +0,0 @@ -BRANCH = "develop" -MASTER_HOST = "buildbot_master_host" -PROJECT_CODE_URL = "project_dirname" -PROJECT_NAME = "{{ project_name }}" -REPOSITORY_URL = "/path/to/repo" - diff --git a/project_template/buildbot_config/slaves/sqlite_slave.py b/project_template/buildbot_config/slaves/sqlite_slave.py index cdb571b..2ad8205 100644 --- a/project_template/buildbot_config/slaves/sqlite_slave.py +++ b/project_template/buildbot_config/slaves/sqlite_slave.py @@ -3,7 +3,7 @@ from buildbot.process.factory import BuildFactory from buildbot.steps.shell import ShellCommand -from buildbot_config.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL nickname = 'sqlite' name = 'slave-%s' % nickname diff --git a/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py b/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py index 6f64d5b..a94f35f 100644 --- a/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py +++ b/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py @@ -3,7 +3,7 @@ from buildbot.process.factory import BuildFactory from buildbot.steps.shell import ShellCommand -from buildbot_config.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL svn_username = 'www-data' svn_password = 'www-d@t@!@#' nickname = 'sqlite' diff --git a/setup.py b/setup.py index 11d40e4..2c463c5 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,14 @@ ], packages = ['project_template' , 'project_template.buildbot_config' + , 'project_template.buildbot_config.schedulers' + , 'project_template.buildbot_config.settings' , 'project_template.buildbot_config.slaves' + , 'project_template.buildbot_config.slaves_svn' , 'project_template.project_name' , 'project_template.project_name.tests' , 'project_template.project_name_project' , 'project_template.project_name_project.settings' - , 'project_template.buildbot_config.slaves_svn' # , 'utilities' , 'proteus' , 'templates' diff --git a/tests/test_complete_master_config.py b/tests/test_complete_master_config.py index cb64778..b589367 100644 --- a/tests/test_complete_master_config.py +++ b/tests/test_complete_master_config.py @@ -4,13 +4,13 @@ class TestCompleteMasterConfig(TestCase): def tearDown(self): if os.path.exists('settings.tmp'): - shutil.copyfile('settings.tmp', 'buildbot_config/settings.py') + shutil.copyfile('settings.tmp', 'buildbot_config/settings/settings.py') os.remove('settings.tmp') os.chdir('../') def setUp(self): os.chdir('project_template') - shutil.copyfile('buildbot_config/settings.py', 'settings.tmp') + shutil.copyfile('buildbot_config/settings/settings.py', 'settings.tmp') def test_complete_master_config(self): # Arrange @@ -20,7 +20,7 @@ def test_complete_master_config(self): os.system('../bin/complete_master_config %s %s' % (ec2_host, repository)) # Assert try: - with open('buildbot_config/settings.py') as stream: + with open('buildbot_config/settings/settings.py') as stream: content = stream.read() self.assertTrue('ec2-50-18-236-118.us-west-1.compute.amazonaws.com' in content) self.assertTrue('git://github.com/juacompe/fluffy.git' in content) From 57d17dfc1fd807f6719551ad8a1680f2751dbfd7 Mon Sep 17 00:00:00 2001 From: cartoonmed Date: Wed, 21 Aug 2013 16:49:29 +0700 Subject: [PATCH 6/6] separate master.cfg to module for support multiple project --- .../buildbot_config/schedulers/__init__.py | 0 .../buildbot_config/schedulers/schedule.py | 21 ++++++++++++++++ .../buildbot_config/settings/__init__.py | 0 .../buildbot_config/settings/settings.py | 6 +++++ .../buildbot_config/slaves/pg_slave.py | 24 +++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 project_template/buildbot_config/schedulers/__init__.py create mode 100644 project_template/buildbot_config/schedulers/schedule.py create mode 100644 project_template/buildbot_config/settings/__init__.py create mode 100644 project_template/buildbot_config/settings/settings.py create mode 100644 project_template/buildbot_config/slaves/pg_slave.py diff --git a/project_template/buildbot_config/schedulers/__init__.py b/project_template/buildbot_config/schedulers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project_template/buildbot_config/schedulers/schedule.py b/project_template/buildbot_config/schedulers/schedule.py new file mode 100644 index 0000000..a4503bd --- /dev/null +++ b/project_template/buildbot_config/schedulers/schedule.py @@ -0,0 +1,21 @@ +from buildbot.changes.gitpoller import GitPoller +from buildbot.changes import filter +from buildbot.schedulers.basic import SingleBranchScheduler + +#select project's settings to buildbot config. +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL, REPOSITORY_URL + +#builder name for this project. +builder_names = ['builder-sqlite', 'builder-pg'] +gitpoller = GitPoller(REPOSITORY_URL + , project=PROJECT_NAME + , branch=BRANCH + , pollinterval=30 + ) +#scheduler for buildbot. +change_filter = filter.ChangeFilter(project=PROJECT_NAME, branch=BRANCH) +scheduler = SingleBranchScheduler(name="develop-change" + , change_filter = change_filter + , treeStableTimer=30 + , builderNames=builder_names + ) diff --git a/project_template/buildbot_config/settings/__init__.py b/project_template/buildbot_config/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project_template/buildbot_config/settings/settings.py b/project_template/buildbot_config/settings/settings.py new file mode 100644 index 0000000..dc4afb1 --- /dev/null +++ b/project_template/buildbot_config/settings/settings.py @@ -0,0 +1,6 @@ +BRANCH = "develop" +MASTER_HOST = "buildbot_master_host" +PROJECT_CODE_URL = "project_dirname" +PROJECT_NAME = "{{ project_name }}" +REPOSITORY_URL = "/path/to/repo" + diff --git a/project_template/buildbot_config/slaves/pg_slave.py b/project_template/buildbot_config/slaves/pg_slave.py new file mode 100644 index 0000000..a301cee --- /dev/null +++ b/project_template/buildbot_config/slaves/pg_slave.py @@ -0,0 +1,24 @@ +from buildbot.buildslave import BuildSlave +from buildbot.config import BuilderConfig +from buildbot.process.factory import BuildFactory +from buildbot.steps.shell import ShellCommand + +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL + +nickname = 'pg' +name = 'slave-%s' % nickname +builder_name = 'builder-%s' % nickname +# slave +slave = BuildSlave(name, "%spassword" % name) +# builder +factory = BuildFactory() +factory.addStep(ShellCommand(command="git pull origin develop", workdir=PROJECT_CODE_URL)) +# Pip install and update to environment which run this buildbot +factory.addStep(ShellCommand(command=["pip", "install", "--upgrade", "--requirement=setup/requirements.txt"],workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["pip", "freeze"], workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["/bin/bash","reset_db"], workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["/bin/bash","runtests", "--settings=%s_project.settings.pg_buildbot" % PROJECT_NAME, "--noinput"], workdir=PROJECT_CODE_URL)) + +builder = BuilderConfig(name=builder_name + , slavenames=[name] + , factory=factory)