Merge pull request #367 from roomorama/release/0.10.0

Release/0.10.0

CHANGELOG.md

@@ -4,6 +4,15 @@ This file summarises the most important changes that went live on each release
 of Concierge. Please check the Wiki entry on the release process to understand
 how this file is formatted and how the process works.
 
+## [0.10.0] - 2016-09-20
+### Added
+- Atleisure:: calendar sync worker
+- reason for all properties skipped during sync
+- page for sync process stats
+
+### Fixed
+- Ciirus:: skip properties without images
+
 ## [0.9.2] - 2016-09-19
 ### Fixed
 - Improper type/subtype combinations

apps/web/config/routes.rb

@@ -1,4 +1,5 @@
-get "/", to: "dashboard#index",       as: :root
+get "/sync_processes/:id/stats", to: "sync_processes#stats", as: :stats
+get "/",                         to: "dashboard#index",      as: :root
 
 resources :errors,         only: [:index, :show], controller: "external_errors"
 resources :reservations,   only: [:index]

apps/web/controllers/external_errors/show.rb

@@ -10,16 +10,7 @@ class Show
     def call(params)
       @error = ExternalErrorRepository.find(params[:id])
 
-      unless @error
-        render_404_template
-      end
-    end
-
-    private
-
-    def render_404_template
-      template = Web::Controllers::TemplateRenderer.new("404.html.erb").render
-      status 404, template
+      halt 404 unless @error
     end
   end
 end

apps/web/controllers/sync_processes/stats.rb

@@ -0,0 +1,14 @@
+module Web::Controllers::SyncProcesses
+  class Stats
+    include Web::Action
+    include Web::Controllers::InternalError
+
+    expose :sync_process
+
+    def call(params)
+      @sync_process = SyncProcessRepository.find(params[:id])
+
+      halt 404 unless @sync_process
+    end
+  end
+end

apps/web/templates/sync_processes/index.html.erb

@@ -21,6 +21,7 @@
         <th>Properties Deleted</th>
         <th>Properties Updated</th>
         <th>Properties Skipped</th>
+        <th>Stats</th>
       </tr>
     </thead>
 
@@ -36,7 +37,8 @@
           <td><%= format_number sync.stats[:properties_created] %></td>
           <td><%= format_number sync.stats[:properties_updated] %></td>
           <td><%= format_number sync.stats[:properties_deleted] %></td>
-          <td><%= format_number sync.stats[:properties_skipped] %></td>
+          <td><%= format_number sync.skipped_properties_count %></td>
+          <td><%= link_to(image("stats16.png"), routes.stats_path(sync.id))%></td>
         <% end %>
       </tr>
     </tbody>

apps/web/templates/sync_processes/stats.html.erb

@@ -0,0 +1,6 @@
+<div class="content">
+  <h2>Sync Process Stats</h2>
+  <pre class="highlight json code-block">
+<%= pretty_print_json(sync_process.stats) %>
+  </pre>
+</div>

apps/web/views/sync_processes/stats.rb

@@ -0,0 +1,26 @@
+module Web::Views::SyncProcesses
+  class Stats
+    include Web::View
+    include Concierge::JSON
+
+    def pretty_print_json(content)
+
+      # uses the +pretty+ and +indent+ options provided by +Yajl::Encoder+ to
+      # format the parsed JSON. Generates two line breaks per line (not for empty arrays)
+      # to make the final content more readable.
+      compact_empty_arrays(
+        double_line_breaks Yajl::Encoder.encode(content.to_h, pretty: true, indent: " " * 2)
+      )
+    end
+
+    private
+
+    def double_line_breaks(str)
+      str.gsub("\n", "\n\n")
+    end
+
+    def compact_empty_arrays(str)
+      str.gsub(/\[\s*\]/, '[]')
+    end
+  end
+end

apps/workers/property_synchronisation.rb

@@ -35,18 +35,19 @@ class PropertySynchronisation
     # Calendar availabilities is tackled by +Workers::CalendarSynchronisation+
     WORKER_TYPE = "metadata"
 
-    PropertyCounters = Struct.new(:created, :updated, :deleted, :skipped)
+    PropertyCounters = Struct.new(:created, :updated, :deleted)
 
-    attr_reader :host, :router, :sync_record, :counters, :processed, :purge
+    attr_reader :host, :router, :sync_record, :counters, :processed, :purge, :properties_skipped
 
     # host - an instance of +Host+.
     def initialize(host)
-      @host        = host
-      @router      = Workers::Router.new(host)
-      @sync_record = init_sync_record(host)
-      @counters    = PropertyCounters.new(0, 0, 0, 0)
-      @processed   = []
-      @purge       = true
+      @host               = host
+      @router             = Workers::Router.new(host)
+      @sync_record        = init_sync_record(host)
+      @counters           = PropertyCounters.new(0, 0, 0)
+      @processed          = []
+      @purge              = true
+      @properties_skipped = Hash.new { |hsh, key| hsh[key] = [] }
     end
 
     # Indicates that the property with the given +identifier+ is being synchronised.
@@ -136,8 +137,8 @@ def finish!
     #   synchronisation.skip_property
     # end
     #
-    def skip_property
-      counters.skipped += 1
+    def skip_property(property_id, reason)
+      properties_skipped[reason] << property_id
     end
 
     # Used to initialize a clean context for a property id.
@@ -210,12 +211,21 @@ def save_sync_process
       sync_record.stats[:properties_created] = counters.created
       sync_record.stats[:properties_updated] = counters.updated
       sync_record.stats[:properties_deleted] = counters.deleted
-      sync_record.stats[:properties_skipped] = counters.skipped
+      sync_record.stats[:properties_skipped] = prepare_properties_skipped
       sync_record.finished_at = Time.now
 
       database.create(sync_record)
     end
 
+    def prepare_properties_skipped
+      properties_skipped.map do |msg, ids|
+        {
+          'reason' => msg,
+          'ids' => ids
+        }
+      end
+    end
+
     def run_operation(operation, *args)
       # enable context tracking when performing API calls to Roomorama so that
       # any errors during the request can be logged.

apps/workers/suppliers/atleisure/availabilities.rb

@@ -0,0 +1,101 @@
+module Workers
+  module Suppliers
+    module AtLeisure
+      # +Workers::Suppliers::AtLeisure::Availabilities+
+      #
+      # Performs properties availabilities synchronisation with supplier
+      class Availabilities
+        BATCH_SIZE = 50
+
+        attr_reader :synchronisation, :host
+
+        def initialize(host)
+          @host            = host
+          @synchronisation = Workers::CalendarSynchronisation.new(host)
+        end
+
+        def perform
+          identifiers = all_identifiers
+
+          identifiers.each_slice(BATCH_SIZE) do |ids|
+            result = synchronisation.new_context { importer.fetch_availabilities(ids) }
+            if result.success?
+              availabilities = result.value
+              availabilities.each do |availability|
+                property_id = availability['HouseCode']
+                synchronisation.start(property_id) do
+                  next availability_error(availability) unless valid_availability?(availability)
+
+                  mapper.build(availability)
+                end
+              end
+            else
+              message = "Failed to perform the `#fetch_availabilities` operation, with properties: `#{ids}`"
+              announce_error(message, result)
+            end
+          end
+          synchronisation.finish!
+        end
+
+        private
+
+        def availability_error(availability)
+          property_id = availability['HouseCode']
+          error_message = availability['error']
+          message = "Error during fetching availabilities for property `#{property_id}`: `#{error_message}`"
+          augment_context_error(message)
+
+          Result.error(:availability_error)
+        end
+
+        def valid_availability?(availability)
+          availability['error'].nil?
+        end
+
+        def all_identifiers
+          PropertyRepository.from_host(host).only(:identifier).map(&:identifier)
+        end
+
+        def mapper
+          @mapper ||= ::AtLeisure::Mappers::Calendar.new
+        end
+
+        def importer
+          @importer ||= ::AtLeisure::Importer.new(credentials)
+        end
+
+        def credentials
+          Concierge::Credentials.for(::AtLeisure::Client::SUPPLIER_NAME)
+        end
+
+        def augment_context_error(message)
+          message = {
+            label: 'Synchronisation Failure',
+            message: message,
+            backtrace: caller
+          }
+          context = Concierge::Context::Message.new(message)
+          Concierge.context.augment(context)
+        end
+
+        def announce_error(message, result)
+          augment_context_error(message)
+
+          Concierge::Announcer.trigger(Concierge::Errors::EXTERNAL_ERROR, {
+            operation:   'sync',
+            supplier:    ::AtLeisure::Client::SUPPLIER_NAME,
+            code:        result.error.code,
+            context:     Concierge.context.to_h,
+            happened_at: Time.now
+          })
+        end
+      end
+    end
+  end
+end
+
+# listen to supplier worker
+Concierge::Announcer.on('availabilities.AtLeisure') do |host, args|
+  Workers::Suppliers::AtLeisure::Availabilities.new(host).perform
+  Result.new({})
+end

apps/workers/suppliers/atleisure.rb → apps/workers/suppliers/atleisure/metadata.rb

@@ -1,9 +1,8 @@
-module Workers::Suppliers
-  # +Workers::Suppliers::AtLeisure+
+module Workers::Suppliers::AtLeisure
+  # +Workers::Suppliers::AtLeisure::Metadata+
   #
-  # Performs synchronisation with supplier
-  class AtLeisure
-    SUPPLIER_NAME = 'AtLeisure'
+  # Performs properties synchronisation with supplier
+  class Metadata
     BATCH_SIZE = 50
 
     attr_reader :synchronisation, :host
@@ -14,7 +13,7 @@ def initialize(host)
     end
 
     def perform
-      result = importer.fetch_properties
+      result = synchronisation.new_context { importer.fetch_properties }
       if result.success?
         grouped_actual_properties(result.value).each do |properties|
           fetch_data_and_process(properties)
@@ -36,19 +35,22 @@ def grouped_actual_properties(properties)
 
     def fetch_data_and_process(properties)
       ids = identifiers(properties)
-      result = importer.fetch_data(ids)
+      result = synchronisation.new_context { importer.fetch_data(ids) }
       if result.success?
         properties_data = result.value
         properties_data.map do |property|
+          property_id = property['HouseCode']
           if validator(property).valid?
-            synchronisation.start(property['HouseCode']) {
+            synchronisation.start(property_id) {
               # AtLeisure's API calls return with large result payloads while
               # synchronising properties, therefore, event tracking is disabled
               # while the property is parsed.
               Concierge.context.disable!
 
               mapper.prepare(property)
             }
+          else
+            synchronisation.skip_property(property_id, 'Invalid property')
           end
         end
       else
@@ -71,7 +73,7 @@ def importer
     end
 
     def credentials
-      Concierge::Credentials.for(SUPPLIER_NAME)
+      Concierge::Credentials.for(AtLeisure::Client::SUPPLIER_NAME)
     end
 
     def mapper
@@ -89,7 +91,7 @@ def announce_error(message, result)
 
       Concierge::Announcer.trigger(Concierge::Errors::EXTERNAL_ERROR, {
         operation:   'sync',
-        supplier:    SUPPLIER_NAME,
+        supplier:    AtLeisure::Client::SUPPLIER_NAME,
         code:        result.error.code,
         context:     Concierge.context.to_h,
         happened_at: Time.now
@@ -100,6 +102,6 @@ def announce_error(message, result)
 
 # listen supplier worker
 Concierge::Announcer.on("metadata.AtLeisure") do |host, args|
-  Workers::Suppliers::AtLeisure.new(host).perform
+  Workers::Suppliers::AtLeisure::Metadata.new(host).perform
   Result.new({})
 end