Closes #96. NLP detects partially matching values

app/models/abstractor/abstractor_suggestion_source_range.rb

@@ -0,0 +1,5 @@
+module Abstractor
+  class AbstractorSuggestionSourceRange < ActiveRecord::Base
+    include Abstractor::Methods::Models::AbstractorSuggestionSourceRange
+  end
+end

db/migrate/20150522134853_create_abstractor_suggestion_source_ranges.rb

@@ -0,0 +1,11 @@
+class CreateAbstractorSuggestionSourceRanges < ActiveRecord::Migration
+  def change
+    create_table :abstractor_suggestion_source_ranges do |t|
+      t.integer :abstractor_suggestion_source_id, null: false
+      t.integer :begin_position
+      t.integer :end_position
+
+      t.timestamps null: false
+    end
+  end
+end

lib/abstractor/methods/models/abstractor_abstraction.rb

@@ -70,6 +70,17 @@ def detect_abstractor_suggestion(suggested_value, unknown, not_applicable)
             end
           end
 
+          def detect_abstractor_suggestion_partial(suggested_value, unknown, not_applicable, begin_position, end_position)
+            abstractor_suggestion = nil
+            abstractor_suggestion = abstractor_suggestions(true).detect do |abstractor_suggestion|
+              m = suggested_value &&
+              Regexp.new(Regexp.escape(suggested_value)) =~ abstractor_suggestion.suggested_value &&
+              abstractor_suggestion.unknown == unknown &&
+              abstractor_suggestion.not_applicable == not_applicable &&
+              abstractor_suggestion.abstractor_suggestion_sources.map(&:abstractor_suggestion_source_ranges).flatten.select{|abstractor_suggestion_source_range| abstractor_suggestion_source_range.begin_position <= begin_position && abstractor_suggestion_source_range.end_position >= end_position}.any?
+            end
+          end
+
           ##
           # Determines if the abstraction has been reviewed.
           #

lib/abstractor/methods/models/abstractor_suggestion_source.rb

@@ -8,8 +8,16 @@ def self.included(base)
           # Associations
           base.send :belongs_to, :abstractor_abstraction_source
           base.send :belongs_to, :abstractor_suggestion
+          base.send :has_many,   :abstractor_suggestion_source_ranges, dependent: :destroy
 
           # base.send :attr_accessible, :abstractor_abstraction_source, :abstractor_abstraction_source_id, :abstractor_suggestion, :abstractor_suggestion_id, :source_id, :source_type, :source_method, :match_value, :deleted_at, :sentence_match_value, :custom_method, :custom_explanation
+          base.send(:include, InstanceMethods)
+        end
+
+        module InstanceMethods
+          def detect_abstractor_suggestion_source_range(begin_position, end_position)
+            abstractor_suggestion_source_ranges.where(begin_position: begin_position, end_position: end_position).first
+          end
         end
       end
     end

lib/abstractor/methods/models/abstractor_suggestion_source_range.rb

@@ -0,0 +1,14 @@
+module Abstractor
+  module Methods
+    module Models
+      module AbstractorSuggestionSourceRange
+        def self.included(base)
+          base.send :include, SoftDelete
+
+          # Associations
+          base.send :belongs_to, :abstractor_suggestion
+        end
+      end
+    end
+  end
+end

lib/abstractor/parser.rb

@@ -29,18 +29,24 @@ def initialize(abstractor_text, options = {})
     end
 
     def scan(token, options = {})
-      options[:word_boundary] = true if options[:word_boundary].nil?
+      options[:word_boundary] = true  if options[:word_boundary].nil?
+      options[:match_data]    = false if options[:match_data].nil?
+
       regular_expression = prepare_token(token, options)
       at = prepare_abstractor_text
       if (regular_expression.nil? || at.nil?)
         []
+      elsif options[:match_data]
+        # http://stackoverflow.com/questions/6804557/how-do-i-get-the-match-data-for-all-occurrences-of-a-ruby-regular-expression-in
+        at.to_enum(:scan,regular_expression).map{ Regexp.last_match }
       else
         at.scan(regular_expression)
       end
     end
 
-    def match(token)
-      regular_expression = prepare_token(token)
+    def match(token, options = {})
+      options[:word_boundary] = true if options[:word_boundary].nil?
+      regular_expression = prepare_token(token, options)
       prepare_abstractor_text.match(regular_expression) unless regular_expression.nil?
     end
 

spec/models/abstractor/abstractor_suggestion_source_range_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+module Abstractor
+  RSpec.describe AbstractorSuggestionSourceRange, :type => :model do
+    pending "add some examples to (or delete) #{__FILE__}"
+  end
+end

spec/models/abstractor/encounter_note_spec.rb

@@ -163,6 +163,7 @@
 
       expect(@encounter_note.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_kps).abstractor_suggestions.first.suggested_value).to eq('kps')
     end
+
     #suggestions
     it "does not create another 'has_karnofsky_performance_status' abstraction suggestion upon re-abstraction (using the canonical name/value format)" do
       @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy.  KPS: 90.')
@@ -227,6 +228,152 @@
       expect(@encounter_note.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_kps).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '90% - Able to carry on normal activity; minor signs or symptoms of disease.'}.size).to eq(1)
     end
 
+    describe 'if match is a part of another matched value' do
+      before(:each) do
+        @h_y_abstractor_abstraction_schema = Abstractor::AbstractorAbstractionSchema.where(
+          predicate: 'has_hoehn_and_yahr_score',
+          display_name: 'Hoehn & Yahr stage',
+          abstractor_object_type_id: @list_object_type,
+          preferred_name: 'H&Y').first_or_create
+
+        ['1', '2', '2.5', '3', '3.5', '4', '5'].each do |value|
+          abstractor_object_value = Abstractor::AbstractorObjectValue.where(value: value.to_s).first_or_create
+          Abstractor::AbstractorAbstractionSchemaObjectValue.where(abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema, abstractor_object_value: abstractor_object_value).first_or_create
+        end
+
+        @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create
+        @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @name_value_rule)
+      end
+
+      describe 'with name/value rule type' do
+        before(:each) do
+          @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create
+          @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @name_value_rule)
+        end
+        describe 'it does not create multiple name/value matches if match is a part of another overlapping matched value' do
+          it "using the canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+
+          it "using the squished canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+
+          it "using the sentential format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5.")
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+        end
+
+        describe 'it creates multiple name/value matches if matches do not overlap' do
+          it "using the canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5. H&Y: 2. H&Y: 5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+
+          it "using the squished canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5. H&Y2 or H&Y5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+
+          it "using the sentential format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5. On another hand H&Y can be 2 or 5.")
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+        end
+      end
+
+      describe 'with value rule type' do
+        before(:each) do
+          @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create
+          @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @value_rule)
+        end
+        describe 'it does not create multiple name/value matches if match is a part of another overlapping matched value' do
+          it "using the canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+
+          it "using the squished canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+
+          it "using the sentential format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5.")
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0)
+          end
+        end
+
+        describe 'it creates multiple name/value matches if matches do not overlap' do
+          it "using the canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5. H&Y: 2. H&Y: 5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+
+          it "using the squished canonical name/value format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5. H&Y2 or H&Y5')
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+
+          it "using the sentential format" do
+            @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5. On another hand H&Y can be 2 or 5.")
+            @encounter_note.abstract
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1)
+            expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1)
+          end
+        end
+      end
+    end
+
     #custom suggestions
     it "creates one 'has_karnofsky_performance_status_date' abstraction suggestion (using a custom rule)", focus: false do
       @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy.  The patient's kps is 90.")