diff --git a/bin/complete_master_config b/bin/complete_master_config index d4a12be..7b814c5 100755 --- a/bin/complete_master_config +++ b/bin/complete_master_config @@ -6,7 +6,7 @@ if [ $# -lt 2 ]; then echo "git_url: URL to the repository" exit 1 else - master_config_file='buildbot_config/settings.py' + master_config_file='buildbot_config/settings/settings.py' ec2_host=$1 git_url=$2 diff --git a/project_template/buildbot_config/master.cfg b/project_template/buildbot_config/master.cfg index d7b2e19..cfea8d2 100644 --- a/project_template/buildbot_config/master.cfg +++ b/project_template/buildbot_config/master.cfg @@ -7,7 +7,7 @@ # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -from buildbot_config.settings import BRANCH, MASTER_HOST, PROJECT_NAME, REPOSITORY_URL +from buildbot_config.settings.settings import MASTER_HOST, PROJECT_NAME c = BuildmasterConfig = {} @@ -36,31 +36,28 @@ c['slavePortnum'] = 9989 # the 'change_source' setting tells the buildmaster how it should find out # about source code changes. Here we point to the buildbot clone of pyflakes. -from buildbot.changes.gitpoller import GitPoller -gitpoller_develop = GitPoller(REPOSITORY_URL - , project=PROJECT_NAME - , branch=BRANCH - , pollinterval=30 - ) -c['change_source'] = [ gitpoller_develop ] +source_module = __import__('buildbot_config.schedulers', fromlist=['buildbot_config', 'schedulers']) +src_modules = list_modules(source_module) + +load_modules(source_module, src_modules) + +c['change_source'] = [ getattr(source_module, source).gitpoller for source in src_modules ] ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build -from buildbot.changes import filter -from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler builder_names = [ getattr(slaves_module, slave).builder_name for slave in modules ] -change_filter = filter.ChangeFilter(project=PROJECT_NAME, branch=BRANCH) -scheduler = SingleBranchScheduler(name="develop-change" - , change_filter = change_filter - , treeStableTimer=30 - , builderNames=builder_names) -c['schedulers'] = [ scheduler ] +scheduler = [ getattr(source_module, schedule).scheduler for schedule in src_modules ] +scheduler.append(ForceScheduler(name="force" + , builderNames=builder_names)) + +c['schedulers'] = scheduler ####### BUILDERS diff --git a/project_template/buildbot_config/master_svn.cfg b/project_template/buildbot_config/master_svn.cfg index 731215c..845f820 100644 --- a/project_template/buildbot_config/master_svn.cfg +++ b/project_template/buildbot_config/master_svn.cfg @@ -54,13 +54,16 @@ c['change_source'] = [ svnpoller_trunk ] # case, just kick off a 'runtests' build from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler builder_names = [ getattr(slaves_module, slave).builder_name for slave in modules ] scheduler = SingleBranchScheduler(name="trunk-change" , branch=None , treeStableTimer=30 , builderNames=builder_names) -c['schedulers'] = [ scheduler ] +force_scheduler = ForceScheduler(name="force" + , builderNames=builder_names) +c['schedulers'] = [ scheduler, force_scheduler ] ####### BUILDERS diff --git a/project_template/buildbot_config/schedulers/__init__.py b/project_template/buildbot_config/schedulers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project_template/buildbot_config/schedulers/schedule.py b/project_template/buildbot_config/schedulers/schedule.py new file mode 100644 index 0000000..a4503bd --- /dev/null +++ b/project_template/buildbot_config/schedulers/schedule.py @@ -0,0 +1,21 @@ +from buildbot.changes.gitpoller import GitPoller +from buildbot.changes import filter +from buildbot.schedulers.basic import SingleBranchScheduler + +#select project's settings to buildbot config. +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL, REPOSITORY_URL + +#builder name for this project. +builder_names = ['builder-sqlite', 'builder-pg'] +gitpoller = GitPoller(REPOSITORY_URL + , project=PROJECT_NAME + , branch=BRANCH + , pollinterval=30 + ) +#scheduler for buildbot. +change_filter = filter.ChangeFilter(project=PROJECT_NAME, branch=BRANCH) +scheduler = SingleBranchScheduler(name="develop-change" + , change_filter = change_filter + , treeStableTimer=30 + , builderNames=builder_names + ) diff --git a/project_template/buildbot_config/settings/__init__.py b/project_template/buildbot_config/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project_template/buildbot_config/settings.py b/project_template/buildbot_config/settings/settings.py similarity index 100% rename from project_template/buildbot_config/settings.py rename to project_template/buildbot_config/settings/settings.py diff --git a/project_template/buildbot_config/slaves/pg_slave.py b/project_template/buildbot_config/slaves/pg_slave.py new file mode 100644 index 0000000..a301cee --- /dev/null +++ b/project_template/buildbot_config/slaves/pg_slave.py @@ -0,0 +1,24 @@ +from buildbot.buildslave import BuildSlave +from buildbot.config import BuilderConfig +from buildbot.process.factory import BuildFactory +from buildbot.steps.shell import ShellCommand + +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL + +nickname = 'pg' +name = 'slave-%s' % nickname +builder_name = 'builder-%s' % nickname +# slave +slave = BuildSlave(name, "%spassword" % name) +# builder +factory = BuildFactory() +factory.addStep(ShellCommand(command="git pull origin develop", workdir=PROJECT_CODE_URL)) +# Pip install and update to environment which run this buildbot +factory.addStep(ShellCommand(command=["pip", "install", "--upgrade", "--requirement=setup/requirements.txt"],workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["pip", "freeze"], workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["/bin/bash","reset_db"], workdir=PROJECT_CODE_URL)) +factory.addStep(ShellCommand(command=["/bin/bash","runtests", "--settings=%s_project.settings.pg_buildbot" % PROJECT_NAME, "--noinput"], workdir=PROJECT_CODE_URL)) + +builder = BuilderConfig(name=builder_name + , slavenames=[name] + , factory=factory) diff --git a/project_template/buildbot_config/slaves/sqlite_slave.py b/project_template/buildbot_config/slaves/sqlite_slave.py index cdb571b..2ad8205 100644 --- a/project_template/buildbot_config/slaves/sqlite_slave.py +++ b/project_template/buildbot_config/slaves/sqlite_slave.py @@ -3,7 +3,7 @@ from buildbot.process.factory import BuildFactory from buildbot.steps.shell import ShellCommand -from buildbot_config.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL nickname = 'sqlite' name = 'slave-%s' % nickname diff --git a/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py b/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py index 6f64d5b..a94f35f 100644 --- a/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py +++ b/project_template/buildbot_config/slaves_svn/sqlite_svn_slave.py @@ -3,7 +3,7 @@ from buildbot.process.factory import BuildFactory from buildbot.steps.shell import ShellCommand -from buildbot_config.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL +from buildbot_config.settings.settings import BRANCH, PROJECT_NAME, PROJECT_CODE_URL svn_username = 'www-data' svn_password = 'www-d@t@!@#' nickname = 'sqlite' diff --git a/proteus/install_buildbot_master_env.py b/proteus/install_buildbot_master_env.py index 0089c1d..aed7837 100644 --- a/proteus/install_buildbot_master_env.py +++ b/proteus/install_buildbot_master_env.py @@ -7,9 +7,9 @@ def install_buildbot_master_env(server, virtenv_path): sudo('easy_install virtualenv') sudo("virtualenv --no-site-packages %s" % (virtenv_path), user="www-data") with prefix("source %s/bin/activate" % (virtenv_path)): - sudo("pip install SQLAlchemy==0.7.9", user="www-data") - sudo("pip install twisted==12.0.0", user="www-data") - sudo("pip install buildbot==0.8.4", user="www-data") + sudo("pip install SQLAlchemy==0.7.10", user="www-data") + sudo("pip install twisted==13.0.0", user="www-data") + sudo("pip install buildbot==0.8.7p1", user="www-data") class Configure(Role): """ diff --git a/proteus/install_buildbot_slave_env.py b/proteus/install_buildbot_slave_env.py index 2f976e5..7540cf9 100644 --- a/proteus/install_buildbot_slave_env.py +++ b/proteus/install_buildbot_slave_env.py @@ -9,8 +9,8 @@ def install_buildbot_slave_env(server, virtenv_path): sudo('easy_install virtualenv') sudo("virtualenv --no-site-packages %s" % (virtenv_path), user="www-data") with prefix("source %s/bin/activate" % (virtenv_path)): - sudo("pip install twisted==12.0.0", user="www-data") - sudo("pip install buildbot-slave==0.8.6", user="www-data") + sudo("pip install twisted==13.0.0", user="www-data") + sudo("pip install buildbot-slave==0.8.7p1", user="www-data") class Configure(Role): """ diff --git a/setup.py b/setup.py index 11d40e4..2c463c5 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,14 @@ ], packages = ['project_template' , 'project_template.buildbot_config' + , 'project_template.buildbot_config.schedulers' + , 'project_template.buildbot_config.settings' , 'project_template.buildbot_config.slaves' + , 'project_template.buildbot_config.slaves_svn' , 'project_template.project_name' , 'project_template.project_name.tests' , 'project_template.project_name_project' , 'project_template.project_name_project.settings' - , 'project_template.buildbot_config.slaves_svn' # , 'utilities' , 'proteus' , 'templates' diff --git a/tests/test_complete_master_config.py b/tests/test_complete_master_config.py index cb64778..b589367 100644 --- a/tests/test_complete_master_config.py +++ b/tests/test_complete_master_config.py @@ -4,13 +4,13 @@ class TestCompleteMasterConfig(TestCase): def tearDown(self): if os.path.exists('settings.tmp'): - shutil.copyfile('settings.tmp', 'buildbot_config/settings.py') + shutil.copyfile('settings.tmp', 'buildbot_config/settings/settings.py') os.remove('settings.tmp') os.chdir('../') def setUp(self): os.chdir('project_template') - shutil.copyfile('buildbot_config/settings.py', 'settings.tmp') + shutil.copyfile('buildbot_config/settings/settings.py', 'settings.tmp') def test_complete_master_config(self): # Arrange @@ -20,7 +20,7 @@ def test_complete_master_config(self): os.system('../bin/complete_master_config %s %s' % (ec2_host, repository)) # Assert try: - with open('buildbot_config/settings.py') as stream: + with open('buildbot_config/settings/settings.py') as stream: content = stream.read() self.assertTrue('ec2-50-18-236-118.us-west-1.compute.amazonaws.com' in content) self.assertTrue('git://github.com/juacompe/fluffy.git' in content)