Merge pull request #5 from Springest/weighted_scenarios

Weighted scenarios.

README.md

@@ -18,6 +18,35 @@ Or install it yourself as:
 
     $ gem install ab_panel
 
+## Upgrading from 0.2.0 to 0.3.0
+
+In this new version we've added weights to different conditions/scenarios. This
+is so that you can rollout certain features slowly. We've also removed the
+original (control scenario) that is added standard.
+
+The only thing you need to do to upgrade is update the ``ab_panel.yml``.
+
+Old:
+
+```yaml
+
+foo:
+  - bar1
+  - bar2
+
+```
+
+New (if you want to keep original or need original):
+
+```yaml
+
+foo:
+  bar1: 2
+  bar2: 2
+  original: 2
+
+```
+
 ## Usage
 
 Create a config file with one or more experiments and conditions.
@@ -26,13 +55,14 @@ In `config/ab_panel.yml`
 
 ```yaml
 my_experiment:
-  - condition_b
-  - condition_c
+  original:    1
+  condition_b: 1
+  condition_c: 1
 ```
 
 Note that this will create 3 conditions:
 
-  1. Original condition (control condition)
+  1. Original condition 
   2. Condition B
   3. Condition C
 

ab_panel.gemspec

@@ -6,8 +6,8 @@ require 'ab_panel/version'
 Gem::Specification.new do |spec|
   spec.name          = "ab_panel"
   spec.version       = AbPanel::VERSION
-  spec.authors       = ["Wouter de Vos", "Mark Mulder"]
-  spec.email         = ["[email protected]", "[email protected]"]
+  spec.authors       = ["Wouter de Vos", "Mark Mulder", "Peter de Ruijter"]
+  spec.email         = ["[email protected]", "[email protected]", "[email protected]"]
   spec.description   = %q{Run A/B test experiments on your Rails 3+ site using Mixpanel as a backend.}
   spec.summary       = %q{Run A/B test experiments on your Rails 3+ site using Mixpanel as a backend.}
   spec.homepage      = "https://github.com/Springest/ab_panel"

lib/ab_panel.rb

@@ -1,4 +1,6 @@
 require 'set'
+require_relative './array'
+
 Dir[File.expand_path(File.join(
   File.dirname(__FILE__),'ab_panel','**','*.rb'))]
     .each {|f| require f}
@@ -37,6 +39,10 @@ def scenarios(experiment)
       config.scenarios experiment
     end
 
+    def weights(experiment)
+      config.weights experiment
+    end
+
     def properties
       @env[:properties]
     end
@@ -83,7 +89,7 @@ def assign_conditions!(already_assigned=nil)
         selected = begin
           already_assigned.send(experiment).condition
         rescue
-          scenarios(experiment)[rand(scenarios(experiment).size)]
+          scenarios(experiment).weighted_sample(weights(experiment))
         end
 
         cs[experiment]["#{selected}?"] = true

lib/ab_panel/config.rb

@@ -12,9 +12,13 @@ def experiments
 
     def scenarios(experiment)
       raise ArgumentError.new( "Fatal: Experiment config not found for #{experiment}" ) unless experiments.include? experiment.to_sym
-      ( settings[experiment.to_sym].map(&:to_sym) + [:original] ).uniq
+      ( settings[experiment.to_sym].keys.map(&:to_sym)).uniq
     end
 
+    def weights(experiment)
+      raise ArgumentError.new( "Fatal: Experiment config not found for #{experiment}" ) unless experiments.include? experiment.to_sym
+      settings[experiment.to_sym].map { |s| s[1] }
+    end
 
     def settings
       @settings ||= YAML.load(

lib/ab_panel/version.rb

@@ -1,3 +1,3 @@
 module AbPanel
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

lib/array.rb

@@ -0,0 +1,25 @@
+class Array
+  def weighted_sample(weights=nil)
+    weights = Array.new(length, 1.0) if weights.nil? || weights.sum == 0
+    total = weights.sum
+
+    # The total sum of weights is multiplied by a random number
+    trigger = Kernel::rand * total
+
+    subtotal = 0
+    result = nil
+
+    # The subtotal is checked agains the trigger. The higher the sum, the higher
+    # the probability of triggering a result.
+    weights.each_with_index do |weight, index|
+      subtotal += weight
+
+      if subtotal > trigger
+        result = self[index]
+        break
+      end
+    end
+    # Returns self.last from current array if result is nil
+    result || last
+  end
+end

spec/ab_panel/config_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe AbPanel::Config do
+  let(:config) { AbPanel::Config.new }
+  before do
+    AbPanel::Config.any_instance.stub(:settings) { { exp1: { scenario1: 25, scenario2: 75 } } }
+  end
+
+  describe '.experiments' do
+    subject { config.experiments }
+    it { should =~ [:exp1] }
+  end
+
+  describe '.weights' do
+    subject { config.weights('exp1') }
+
+    it { should =~ [75.0, 25.0] }
+  end
+end

spec/ab_panel_spec.rb

@@ -7,14 +7,29 @@
     it { should =~ %w(experiment1 experiment2).map(&:to_sym) }
   end
 
+  describe ".weights" do
+    let(:experiment) { AbPanel.experiments.first }
+    subject { AbPanel.weights(experiment) }
+
+    it { should == [25, 25, 25, 25] }
+
+    describe "With a nonexistent experiment" do
+      let(:experiment) { :does_not_exist }
+
+      it 'should throw an ArgumentError' do
+        expect { subject }.to raise_exception ArgumentError
+      end
+    end
+  end
+
   describe ".scenarios" do
     subject { AbPanel.scenarios(experiment) }
 
     let(:experiment) { AbPanel.experiments.first }
 
     it { should =~ %w( scenario1 scenario2 scenario3 original ).map(&:to_sym) }
 
-    describe "With an unexisting experiment" do
+    describe "With an nonexistent experiment" do
       let(:experiment) { :does_not_exist }
 
       it 'should throw an ArgumentError' do

spec/array_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe Array do
+  describe '.weighted_sample' do
+    before do
+      Kernel.stub(:rand) { 0.5 }
+    end
+
+    context "Stub test" do
+      subject { Kernel.rand }
+      it { should eq 0.5 }
+    end
+
+    let(:array) { [1, 2, 3, 4] }
+    subject { array.weighted_sample }
+
+    it { should eq 3 }
+
+    context "different random" do
+      before do
+        Kernel.stub(:rand) { 0 }
+      end
+
+      it { should eq 1 }
+    end
+
+    context "different random" do
+      before do
+        Kernel.stub(:rand) { 1 }
+      end
+
+      it { should eq 4 }
+    end
+
+    context "with weights" do
+      subject { array.weighted_sample([1, 0, 0, 0]) }
+      it { should eq 1 }
+    end
+
+    context "all the same weights" do
+      before { Kernel.stub(:rand) { 1 } }
+      subject { array.weighted_sample([0, 0, 0, 0]) }
+      it { should eq 4 }
+      context "random 0" do
+        before { Kernel.stub(:rand) { 0 } }
+        it { should eq 1 }
+      end
+    end
+  end
+end

spec/support/files/config/ab_panel.yml

@@ -1,9 +1,11 @@
 experiment1:
-  - scenario1
-  - scenario2
-  - scenario3
+  scenario1: 25
+  scenario2: 25
+  scenario3: 25
+  original: 25
 
 experiment2:
-  - scenario4
-  - scenario5
+  scenario4: 33.4
+  scenario5: 33.3
+  original: 33.3