Add application cookbook

README.txt

@@ -0,0 +1,170 @@
+Application cookbook
+====================
+
+This cookbook is initially designed to be able to describe and deploy Rails applications. Other application stacks will be available as recipes in later releases.
+
+---
+Recipes
+=======
+
+The application cookbook contains the following recipes.
+
+default
+-------
+
+Searches the `apps` data bag and checks that a server role in the app exists on this node, adds the app to the run state and uses the role for the app to locate the recipes that need to be used.
+
+rails
+-----
+
+Using the node's `run_state` that contains the current application in the search, this recipe will install required packages and gems, set up the deployment scaffolding, creates database and memcached configurations if required and then performs a revision-based deploy.
+
+unicorn
+-------
+
+Unicorn is installed, default attributes are set for the node and an app specific unicorn config and runit service are created.
+
+---
+Deprecated Recipes
+==================
+
+The following recipes are deprecated in favor of rails+unicorn, as that is performant enough for many Rails applications, and takes less time to provision new instances. Using these recipes may require additional work to the rest of the stack that wouldn't be required with rails+unicorn because they were early-phase development of this cookbook.
+
+passenger-nginx
+---------------
+
+Builds passenger as an nginx module, drops off the configuration file and sets up the site to run the application under nginx with passenger. Does not deploy the code automatically.
+
+rails_nginx_ree_passenger
+-------------------------
+
+Sets up the application stack with Ruby Enterprise Edition, Nginx and Passenger.
+
+The recipe searches the apps data bag and then installs packages and gems, creates the nginx vhost config and enables the site, sets up the deployment scaffolding, and uses a revision-based deploy for the code. Database and memcached yaml files are written out as well, if required.
+
+---
+Application Data Bag
+====================
+
+The applications data bag expects some certain values in order to configure parts of the recipe. Below is a paste of the JSON, where the value is a description of the key. Use your own values, as required. Note that this data bag is also used by the `database` cookbook, so it will contain database information as well. Items that may be ambiguous have an example.
+
+The application used in examples is named `my_app` and the environment is `production`.
+
+Note about `databases`, the data specified will be rendered as the `database.yml` file.
+
+Note about gems and packages, the version is optional. If specified, it will lock in on that version, otherwise it will use the latest available.
+
+    {
+      "id": "my_app",
+      "server_roles": [
+        "application specific role(s), typically the name of the app, e.g., my_app",
+        "my_app"
+      ],
+      "type": {
+        "my_app": [
+          "recipes in application cookbook to run for this role",
+          "rails",
+          "unicorn"
+        ]
+      },
+      "memcached_role": [
+        "name of the role used for the app-specific memcached server",
+        "my_app_memcached"
+      ],
+      "database_slave_role": [
+        "name of the role used by database slaves, typically named after the app, 'my_app_database_slave'",
+        "my_app_database_slave"
+      ],
+      "database_master_role": [
+        "name of the role used by database master, typically named after the app 'my_app_database_master'",
+        "my_app_database_master"
+      ],
+      "repository": "git@github.com:company/my_app.git",
+      "revision": {
+        "production": "branch or tag to deploy"
+      },
+      "force": {
+        "production": "true or false w/o quotes to force deployment, see the rails.rb recipe"
+      },
+      "migrate": {
+        "production": "true or false boolean to force migration, see rails.rb recipe"
+      },
+      "databases": {
+        "db_name_production": {
+          "reconnect": "true",
+          "encoding": "utf8",
+          "username": "db_user",
+          "adapter": "mysql",
+          "password": "awesome_password",
+          "database": "db_name_production"
+        }
+      },
+      "mysql_root_password": {
+        "production": "password for the root user in mysql"
+      },
+      "mysql_debian_password": {
+        "production": "password for the debian-sys-maint user on ubuntu/debian"
+      },
+      "mysql_repl_password": {
+        "production": "password for the 'repl' user for replication."
+      },
+      "snapshots_to_keep": {
+        "production": "integer of the number of snapshots we're going to keep for this environment."
+      },
+      "deploy_key": "SSH private key used to deploy from a private git repository",
+      "deploy_to": "path to deploy, e.g. /srv/my_app",
+      "owner": "owner for the application files when deployed",
+      "group": "group for the application files when deployed",
+      "packages": {
+        "package_name": "specific packages required for installation at the OS level to run the app like libraries and specific version, e.g.",
+        "curl": "7.19.5-1ubuntu2"
+      },
+      "gems": {
+        "gem_name": "specific gems required for installation to run the application, and if a specific version is required, e.g.",
+        "rails": "2.3.5"
+      },
+      "memcached": {
+        "production": {
+          "namespace": "specify the memcache namespace, ie my_app_environment"
+        }
+      }
+    }
+
+---
+Usage
+=====
+
+To use the application cookbook, we recommend creating a role named after the application, e.g. `my_app`. In your chef-repo, create `roles/my_app.rb`.
+
+    name "my_app"
+    description "My application front end server."
+    run_list(
+      "recipe[application]"
+    )
+
+If you need other recipes, such as `mysql::client` add those as well. Then upload the role to the Chef Server.
+
+    % knife role from file roles/my_app.rb
+
+Add the role to a node, and watch it deploy the application!
+
+---
+License and Author
+==================
+
+Author:: Adam Jacob (<adam@opscode.com>)
+Author:: Joshua Timberman (<joshua@opscode.com>)
+
+Copyright 2009-2010, Opscode, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

metadata.rb

@@ -0,0 +1,10 @@
+maintainer       "Opscode, Inc."
+maintainer_email "ops@opscode.com"
+license          "Apache 2.0"
+description      "Deploys and configures a Rails application from databag 'apps'"
+long_description  IO.read(File.join(File.dirname(__FILE__), 'README.txt'))
+version          "0.3"
+
+%w{ ruby_enterprise passenger_enterprise runit unicorn }.each do |cb|
+  depends cb
+end

recipes/default.rb

@@ -0,0 +1,30 @@
+#
+# Cookbook Name:: application
+# Recipe:: default
+#
+# Copyright 2009, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+search(:apps) do |app|
+  (app["server_roles"] & node.run_list.roles).each do |app_role|
+    app["type"][app_role].each do |thing|
+      node.run_state[:current_app] = app
+      include_recipe "application::#{thing}"
+    end
+  end
+end
+
+node.run_state.delete(:current_app)
+

recipes/passenger-nginx.rb

@@ -0,0 +1,46 @@
+#
+# Cookbook Name:: application
+# Recipe:: default
+#
+# Copyright 2009, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+app = node.run_state[:current_app] 
+
+include_recipe "passenger_enterprise::nginx"
+
+template "#{node[:nginx][:dir]}/sites-available/#{app['id']}.conf" do
+  source "rails_nginx_passenger.conf.erb"
+  owner "root"
+  group "root"
+  mode "0644"
+  variables(
+    :app => app['id'],
+    :docroot => "/srv/#{app['id']}/current/public",
+    :server_name => "#{app['id']}.#{node[:domain]}",
+    :server_aliases => [ node[:fqdn], app['id'] ],
+    :rails_env => app['environment']
+  )
+end
+
+nginx_site "#{app['id']}.conf" do
+  notifies :restart, resources(:service => "nginx")
+end
+
+d = resources(:deploy => app['id'])
+d.restart_command do
+  service "nginx" do action :restart; end
+end
+

recipes/rails.rb

@@ -0,0 +1,184 @@
+#
+# Cookbook Name:: application
+# Recipe:: default
+#
+# Copyright 2009, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+app = node.run_state[:current_app]
+
+###
+# You really most likely don't want to run this recipe from here - let the
+# default application recipe work it's mojo for you.
+###
+
+# Are we using REE?
+use_ree = false
+if node.run_state[:seen_recipes].has_key?("ruby_enterprise")
+  use_ree = true
+end
+
+node.default[:apps][app['id']][node.app_environment][:run_migrations] = false
+
+## First, install any application specific packages
+if app['packages']
+  app['packages'].each do |pkg,ver|
+    package pkg do
+      action :install
+      version ver if ver && ver.length > 0
+    end
+  end
+end
+
+## Next, install any application specific gems
+if app['gems']
+  app['gems'].each do |gem,ver|
+    if use_ree
+      ree_gem gem do
+        action :install
+        version ver if ver && ver.length > 0
+      end
+    else
+      gem_package gem do
+        action :install
+        version ver if ver && ver.length > 0
+      end
+    end
+  end
+end
+
+directory app['deploy_to'] do
+  owner app['owner']
+  group app['group']
+  mode '0755'
+  recursive true
+end
+
+directory "#{app['deploy_to']}/shared" do
+  owner app['owner']
+  group app['group']
+  mode '0755'
+  recursive true
+end
+
+directory "#{app['deploy_to']}/shared/log" do
+  owner app['owner']
+  group app['group']
+  mode '0755'
+  recursive true
+end
+
+directory "#{app['deploy_to']}/shared/pids" do
+  owner app['owner']
+  group app['group']
+  mode '0755'
+  recursive true
+end
+
+if app.has_key?("deploy_key")
+  ruby_block "write_key" do
+    block do
+      f = File.open("#{app['deploy_to']}/id_deploy", "w")
+      f.print(app["deploy_key"])
+      f.close
+    end
+    not_if do File.exists?("#{app['deploy_to']}/id_deploy"); end
+  end
+
+  file "#{app['deploy_to']}/id_deploy" do
+    owner app['owner']
+    group app['group']
+    mode '0600'
+  end
+
+  template "#{app['deploy_to']}/deploy-ssh-wrapper" do
+    source "deploy-ssh-wrapper.erb"
+    owner app['owner']
+    group app['group']
+    mode "0755"
+    variables app.to_hash
+  end
+end
+
+if app["database_master_role"]
+  dbm = nil
+  # If we are the database master
+  if node.run_list.roles.include?(app["database_master_role"][0])
+    dbm = node
+  else
+  # Find the database master
+    results = search(:node, "run_list:role\\[#{app["database_master_role"][0]}\\] AND app_environment:#{node[:app_environment]}", nil, 0, 1)
+    rows = results[0]
+    if rows.length == 1
+      dbm = rows[0]
+    end
+  end
+
+  # Assuming we have one...
+  if dbm
+    template "#{app['deploy_to']}/shared/database.yml" do
+      source "database.yml.erb"
+      owner app["owner"]
+      group app["group"]
+      mode "644"
+      variables(
+        :host => dbm['fqdn'],
+        :databases => app['databases']
+      )
+    end
+  else
+    Chef::Log.warn("No node with role #{app["database_master_role"][0]}, database.yml not rendered!")
+  end
+end
+
+if app["memcached_role"]
+  results = search(:node, "role:#{app["memcached_role"][0]} AND app_environment:#{node[:app_environment]} NOT hostname:#{node[:hostname]}")
+  if results.length == 0
+    if node.run_list.roles.include?(app["memcached_role"][0])
+      results << node
+    end
+  end
+  template "#{app['deploy_to']}/shared/memcached.yml" do
+    source "memcached.yml.erb"
+    owner app["owner"]
+    group app["group"]
+    mode "644"
+    variables(
+      :memcached_envs => app['memcached'],
+      :hosts => results
+    )
+  end
+end
+
+## Then, deploy
+deploy_revision app['id'] do
+  revision app['revision'][node.app_environment]
+  repository app['repository']
+  user app['owner']
+  group app['group']
+  deploy_to app['deploy_to']
+  action app['force'][node.app_environment] ? :force_deploy : :deploy
+  ssh_wrapper "#{app['deploy_to']}/deploy-ssh-wrapper" if app['deploy_key']
+  if app['migrate'][node.app_environment] && node[:apps][app['id']][node.app_environment][:run_migrations]
+    migrate true
+    migration_command "rake db:migrate"
+  else
+    migrate false
+  end
+  symlink_before_migrate({
+    "database.yml" => "config/database.yml",
+    "memcached.yml" => "config/memcached.yml"
+  })
+end

recipes/rails_nginx_ree_passenger.rb

@@ -0,0 +1,178 @@
+#
+# Cookbook Name:: application
+# Recipe:: default
+#
+# Copyright 2009, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "ruby_enterprise"
+
+runit_service "nginx"
+
+search(:apps) do |app|
+  if (app["server_roles"] & node.run_list.roles).length > 0
+
+    node.default[:apps][app['id']][node.app_environment][:run_migrations] = false
+
+    ## First, install any application specific packages
+    if app['packages']
+      app['packages'].each do |pkg,ver|
+        package pkg do
+          action :install
+          version ver if ver && ver.length > 0
+        end
+      end
+    end
+
+    ## Next, install any application specific gems
+    if app['gems']
+      app['gems'].each do |gem,ver|
+        ree_gem gem do
+          action :install
+          version ver if ver && ver.length > 0
+        end
+      end
+    end
+
+    ## Then, configure nginx
+    template "#{node[:nginx][:dir]}/sites-available/#{app['id']}.conf" do
+      source "rails_nginx_passenger.conf.erb"
+      owner "root"
+      group "root"
+      mode "0644"
+      variables(
+        :app => app['id'],
+        :docroot => "/srv/#{app['id']}/current/public",
+        :server_name => "#{app['id']}.#{node[:domain]}",
+        :server_aliases => [ node[:fqdn], app['id'] ],
+        :rails_env => app['environment']
+      )
+    end
+
+    nginx_site "#{app['id']}.conf" do
+      notifies :restart, resources(:service => "nginx")
+    end
+
+    directory app['deploy_to'] do
+      owner app['owner']
+      group app['group']
+      mode '0755'
+      recursive true
+    end
+
+    directory "#{app['deploy_to']}/shared" do
+      owner app['owner']
+      group app['group']
+      mode '0755'
+      recursive true
+    end
+    
+    directory "#{app['deploy_to']}/shared/log" do
+      owner app['owner']
+      group app['group']
+      mode '0755'
+      recursive true
+    end
+
+    if app.has_key?("deploy_key")
+      ruby_block "write_key" do
+        block do
+          f = File.open("#{app['deploy_to']}/id_deploy", "w")
+          f.print(app["deploy_key"])
+          f.close
+        end
+        not_if do File.exists?("#{app['deploy_to']}/id_deploy"); end
+      end
+
+      file "#{app['deploy_to']}/id_deploy" do
+        owner app['owner']
+        group app['group']
+        mode '0600'
+      end
+
+      template "#{app['deploy_to']}/deploy-ssh-wrapper" do
+        source "deploy-ssh-wrapper.erb"
+        owner app['owner']
+        group app['group']
+        mode "0755"
+        variables app.to_hash
+      end
+    end
+
+    ## Then, deploy
+    deploy_revision app['id'] do
+      revision app['revision'][node.app_environment]
+      repository app['repository']
+      user app['owner']
+      group app['group']
+      deploy_to app['deploy_to']
+      action app['force'][node.app_environment] ? :force_deploy : :deploy
+      ssh_wrapper "#{app['deploy_to']}/deploy-ssh-wrapper" if app['deploy_key']
+      if app['migrate'][node.app_environment] && node[:apps][app['id']][node.app_environment][:run_migrations]
+        migrate true
+        migration_command "rake db:migrate"
+      else
+        migrate false
+      end
+      restart_command do
+        case app["type"]
+        when /nginx/
+          service "nginx" do action :restart; end
+        when /apache/
+          service "apache" do action :restart; end
+        end
+      end
+      symlink_before_migrate({ 
+        "database.yml" => "config/database.yml",
+        "memcached.yml" => "config/memcached.yml"
+      })
+      before_symlink do
+        if app["database_master_role"]
+          results = search(:node, "run_list:role\\[#{app["database_master_role"][0]}\\]", nil, 0, 1)
+          rows = results[0]
+          if rows.length == 1
+            dbm = rows[0]
+            template "#{@new_resource.shared_path}/database.yml" do
+              source "database.yml.erb"
+              owner app["owner"]
+              group app["group"]
+              mode "644"
+              variables(
+                :host => dbm['fqdn'],
+                :databases => app['databases']
+              )
+            end
+          else
+            Chef::Log.warn("No node with role #{app["database_master_role"][0]}, database.yml not rendered!")
+          end
+        end
+        if app["memcached_role"]
+          results = search(:node, "run_list:role\\[#{app["memcached_role"][0]}\\]")
+          rows = results[0]
+          template "#{@new_resource.shared_path}/memcached.yml" do
+            source "memcached.yml.erb"
+            owner app["owner"]
+            group app["group"]
+            mode "644"
+            variables(
+              :memcached_envs => app['memcached'],
+              :hosts => rows
+            )
+          end
+        end
+      end
+    end
+  end
+end

recipes/unicorn.rb

@@ -0,0 +1,52 @@
+#
+# Cookbook Name:: application
+# Recipe:: unicorn 
+#
+# Copyright 2009, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+app = node.run_state[:current_app] 
+
+include_recipe "unicorn"
+
+node.default[:unicorn][:worker_timeout] = 60
+node.default[:unicorn][:preload_app] = false
+node.default[:unicorn][:worker_processes] = node[:cpu][:total].to_i * 4 
+node.default[:unicorn][:preload_app] = false
+node.default[:unicorn][:before_fork] = 'sleep 1' 
+node.default[:unicorn][:port] = '8080'
+node.set[:unicorn][:options] = { :tcp_nodelay => true, :backlog => 100 }
+
+unicorn_config "/etc/unicorn/#{app['id']}.rb" do
+  listen({ node[:unicorn][:port] => node[:unicorn][:options] })
+  worker_timeout node[:unicorn][:worker_timeout] 
+  preload_app node[:unicorn][:preload_app] 
+  worker_processes node[:unicorn][:worker_processes]
+  before_fork node[:unicorn][:before_fork] 
+end
+
+runit_service app['id'] do
+  template_name 'unicorn'
+  cookbook 'application'
+  options(:app => app)
+end
+
+if File.exists?(File.join(app['deploy_to'], "current"))
+  d = resources(:deploy => app['id'])
+  d.restart_command do
+    execute "/etc/init.d/#{app['id']} hup"
+  end
+end
+

templates/default/database.yml.erb

@@ -0,0 +1,10 @@
+<%- @databases.each do |env, db| %>
+<%= env %>:
+  adapter: <%= db['adapter'] %>
+  host: <%= @host %>
+  database: <%= db['database'] %>
+  username: <%= db['username'] %>
+  password: <%= db['password'] %>
+  encoding: <%= db.has_key?('encoding') ? db['encoding'] : 'utf8' %>
+  reconnect: true
+<%- end %>

templates/default/deploy-ssh-wrapper.erb

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# 
+# Deploy SSH Wrapper
+# App: <%= @id %> 
+#
+# Rendered by Chef - local changes will be replaced
+
+/usr/bin/env ssh -o "StrictHostKeyChecking=no" -i "<%= @deploy_to %>/id_deploy" $1 $2

templates/default/memcached.yml.erb

@@ -0,0 +1,10 @@
+<%- @memcached_envs.each do |env, settings| %>
+<%= env %>:
+<%- settings.each do |k, v| %>
+  <%= k %>: <%= v %>
+<%- end %>
+  servers:
+<%- @hosts.each do |h| %>
+    - <%= h['ipaddress'] %>:<%= h['memcached']['port'] %>
+<%- end %>
+<%- end %>

templates/default/myproj.conf.erb

@@ -0,0 +1,13 @@
+# nginx + passenger application vhost
+#
+# Generated by Chef for <%= @node[:fqdn] %>
+# Local modifications will be overwritten.
+#
+server {
+  listen 80;
+  server_name <%= @server_name %><% @server_aliases.each do |a| %><%= " #{a}" %> <% end %>;
+  root <%= @docroot %>;
+  error_log <%= @node[:nginx][:log_dir] %>/<%= @app %>-error.log;
+  access_log <%= @node[:nginx][:log_dir] %>/<%= @app %>-access.log;
+  passenger_enabled on;
+}

templates/default/rails_nginx_passenger.conf.erb

@@ -0,0 +1,13 @@
+# nginx + passenger application vhost
+#
+# Generated by Chef for <%= @node[:fqdn] %>
+# Local modifications will be overwritten.
+#
+server {
+  listen 80;
+  server_name <%= @server_name %><% @server_aliases.each do |a| %><%= " #{a}" %> <% end %>;
+  root <%= @docroot %>;
+  error_log <%= @node[:nginx][:log_dir] %>/<%= @app %>-error.log;
+  access_log <%= @node[:nginx][:log_dir] %>/<%= @app %>-access.log;
+  passenger_enabled on;
+}

templates/default/sv-unicorn-log-run.erb

@@ -0,0 +1,3 @@
+#!/bin/sh
+exec svlogd -tt ./main
+

templates/default/sv-unicorn-run.erb

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd <%= @options[:app]['deploy_to'] %>/current
+
+exec 2>&1
+exec chpst -u <%= @options[:app]["owner"] %>:<%= @options[:app]["group"] %> unicorn_rails -E <%= @node[:app_environment] %> -c /etc/unicorn/<%= @options[:app]['id'] %>.rb
+