From ffa0b80dc6d27151de73caa75f54351b8e96e356 Mon Sep 17 00:00:00 2001 From: Yulia Bushmanova Date: Wed, 29 Apr 2015 13:27:22 -0500 Subject: [PATCH] Closes #96. NLP detects partially matching values --- .../abstractor_suggestion_source_range.rb | 5 + ...ate_abstractor_suggestion_source_ranges.rb | 11 ++ .../methods/models/abstractor_abstraction.rb | 11 ++ .../methods/models/abstractor_subject.rb | 115 +++++++++----- .../models/abstractor_suggestion_source.rb | 8 + .../abstractor_suggestion_source_range.rb | 14 ++ lib/abstractor/parser.rb | 12 +- ...abstractor_suggestion_source_range_spec.rb | 7 + spec/models/abstractor/encounter_note_spec.rb | 147 ++++++++++++++++++ .../radiation_therapy_prescription_spec.rb | 2 +- 10 files changed, 285 insertions(+), 47 deletions(-) create mode 100644 app/models/abstractor/abstractor_suggestion_source_range.rb create mode 100644 db/migrate/20150522134853_create_abstractor_suggestion_source_ranges.rb create mode 100644 lib/abstractor/methods/models/abstractor_suggestion_source_range.rb create mode 100644 spec/models/abstractor/abstractor_suggestion_source_range_spec.rb diff --git a/app/models/abstractor/abstractor_suggestion_source_range.rb b/app/models/abstractor/abstractor_suggestion_source_range.rb new file mode 100644 index 0000000..92066d5 --- /dev/null +++ b/app/models/abstractor/abstractor_suggestion_source_range.rb @@ -0,0 +1,5 @@ +module Abstractor + class AbstractorSuggestionSourceRange < ActiveRecord::Base + include Abstractor::Methods::Models::AbstractorSuggestionSourceRange + end +end \ No newline at end of file diff --git a/db/migrate/20150522134853_create_abstractor_suggestion_source_ranges.rb b/db/migrate/20150522134853_create_abstractor_suggestion_source_ranges.rb new file mode 100644 index 0000000..796fb0f --- /dev/null +++ b/db/migrate/20150522134853_create_abstractor_suggestion_source_ranges.rb @@ -0,0 +1,11 @@ +class CreateAbstractorSuggestionSourceRanges < ActiveRecord::Migration + def change + create_table :abstractor_suggestion_source_ranges do |t| + t.integer :abstractor_suggestion_source_id, null: false + t.integer :begin_position + t.integer :end_position + + t.timestamps null: false + end + end +end diff --git a/lib/abstractor/methods/models/abstractor_abstraction.rb b/lib/abstractor/methods/models/abstractor_abstraction.rb index 7703918..0065912 100644 --- a/lib/abstractor/methods/models/abstractor_abstraction.rb +++ b/lib/abstractor/methods/models/abstractor_abstraction.rb @@ -70,6 +70,17 @@ def detect_abstractor_suggestion(suggested_value, unknown, not_applicable) end end + def detect_abstractor_suggestion_partial(suggested_value, unknown, not_applicable, begin_position, end_position) + abstractor_suggestion = nil + abstractor_suggestion = abstractor_suggestions(true).detect do |abstractor_suggestion| + m = suggested_value && + Regexp.new(Regexp.escape(suggested_value)) =~ abstractor_suggestion.suggested_value && + abstractor_suggestion.unknown == unknown && + abstractor_suggestion.not_applicable == not_applicable && + abstractor_suggestion.abstractor_suggestion_sources.map(&:abstractor_suggestion_source_ranges).flatten.select{|abstractor_suggestion_source_range| abstractor_suggestion_source_range.begin_position <= begin_position && abstractor_suggestion_source_range.end_position >= end_position}.any? + end + end + ## # Determines if the abstraction has been reviewed. # diff --git a/lib/abstractor/methods/models/abstractor_subject.rb b/lib/abstractor/methods/models/abstractor_subject.rb index 22b0365..5f657bc 100644 --- a/lib/abstractor/methods/models/abstractor_subject.rb +++ b/lib/abstractor/methods/models/abstractor_subject.rb @@ -133,7 +133,7 @@ def abstract_nlp_suggestion(about, abstractor_abstraction, abstractor_abstractio def abstract_custom_suggestion(about, abstractor_abstraction, abstractor_abstraction_source) suggestions = about.send(abstractor_abstraction_source.custom_method, abstractor_abstraction) suggestions.each do |suggestion| - suggest(abstractor_abstraction, abstractor_abstraction_source, nil, nil, about.id, about.class.to_s, abstractor_abstraction_source.from_method, abstractor_abstraction_source.section_name, suggestion[:suggestion], nil, nil, abstractor_abstraction_source.custom_method, suggestion[:explanation]) + suggest(abstractor_abstraction, abstractor_abstraction_source, nil, nil, about.id, about.class.to_s, abstractor_abstraction_source.from_method, abstractor_abstraction_source.section_name, suggestion[:suggestion], nil, nil, abstractor_abstraction_source.custom_method, suggestion[:explanation], nil, nil) end create_unknown_abstractor_suggestion(about, abstractor_abstraction, abstractor_abstraction_source) end @@ -215,21 +215,25 @@ def abstract_sentential_value(about, abstractor_abstraction, abstractor_abstract end parser = Abstractor::Parser.new(abstractor_text) + all_object_variants = {} abstractor_object_values.each do |abstractor_object_value| object_variants(abstractor_object_value, abstractor_object_value_variants).each do |object_variant| - ranges = parser.range_all(Regexp.escape(object_variant.downcase)) - if ranges.any? - ranges.each do |range| - sentence = parser.find_sentence(range) - if sentence - scoped_sentence = Abstractor::NegationDetection.parse_negation_scope(sentence[:sentence]) - reject = ( - Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], object_variant) || - Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], object_variant) - ) - if !reject - suggest(abstractor_abstraction, abstractor_abstraction_source, object_variant.downcase, sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil) - end + all_object_variants[object_variant] = abstractor_object_value + end + end + all_object_variants.sort{|a,b| b[0].length <=> a[0].length}.each do |object_variant, abstractor_object_value| + ranges = parser.range_all(Regexp.escape(object_variant.downcase)) + if ranges.any? + ranges.each do |range| + sentence = parser.find_sentence(range) + if sentence + scoped_sentence = Abstractor::NegationDetection.parse_negation_scope(sentence[:sentence]) + reject = ( + Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], object_variant) || + Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], object_variant) + ) + if !reject + suggest(abstractor_abstraction, abstractor_abstraction_source, object_variant.downcase, sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil, range.begin, range.end) end end end @@ -249,19 +253,28 @@ def abstract_canonical_name_value(about, abstractor_abstraction, abstractor_abst abstractor_abstraction_source.normalize_from_method_to_sources(about).each do |source| abstractor_text = Abstractor::AbstractorAbstractionSource.abstractor_text(source) parser = Abstractor::Parser.new(abstractor_text) + abstractor_abstraction_schema.predicate_variants.each do |predicate_variant| + all_object_variants = {} abstractor_abstraction_schema.abstractor_object_values.each do |abstractor_object_value| - abstractor_object_value.object_variants.each do |object_variant| - match_value = "#{Regexp.escape(predicate_variant)}:\s*#{Regexp.escape(object_variant)}" - matches = parser.scan(match_value, word_boundary: true).uniq - matches.each do |match| - suggest(abstractor_abstraction, abstractor_abstraction_source, match, match, source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil) - end + object_variants(abstractor_object_value, abstractor_object_value.abstractor_object_value_variants).each do |object_variant| + all_object_variants[object_variant] = abstractor_object_value + end + end - match_value = "#{Regexp.escape(predicate_variant)}#{Regexp.escape(object_variant)}" - matches = parser.scan(match_value, word_boundary: true).uniq - matches.each do |match| - suggest(abstractor_abstraction, abstractor_abstraction_source, match, match, source[:source_id], source[:source_type].to_s, source[:source_method], source[:seciton_name], abstractor_object_value, nil, nil, nil, nil) + all_object_variants.sort{|a,b| b[0].to_s.length <=> a[0].to_s.length}.each do |object_variant, abstractor_object_value| + match_value = "#{Regexp.escape(predicate_variant)}:\s*#{Regexp.escape(object_variant)}" + matches = parser.scan(match_value, word_boundary: true, match_data: true) + if matches + for i in (0...matches.size) do + suggest(abstractor_abstraction, abstractor_abstraction_source, matches[i].to_s, matches[i].to_s, source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil, matches[i].begin(0), matches[i].end(0)) + end + end + match_value = "#{Regexp.escape(predicate_variant)}#{Regexp.escape(object_variant)}" + matches = parser.scan(match_value, word_boundary: true, match_data: true) + if matches + for i in (0...matches.size) do + suggest(abstractor_abstraction, abstractor_abstraction_source, matches[i].to_s, matches[i].to_s, source[:source_id], source[:source_type].to_s, source[:source_method], source[:seciton_name], abstractor_object_value, nil, nil, nil, nil, matches[i].begin(0), matches[i].end(0)) end end end @@ -279,20 +292,25 @@ def abstract_sentential_name_value(about, abstractor_abstraction, abstractor_abs ranges.each do |range| sentence = parser.find_sentence(range) if sentence + all_object_variants = {} abstractor_abstraction_schema.abstractor_object_values.each do |abstractor_object_value| - abstractor_object_value.object_variants.each do |object_variant| - match = parser.match_sentence(sentence[:sentence], Regexp.escape(object_variant)) - if match - scoped_sentence = Abstractor::NegationDetection.parse_negation_scope(sentence[:sentence]) - reject = ( - Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], predicate_variant) || - Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], predicate_variant) || - Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], object_variant) || - Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], object_variant) - ) - if !reject - suggest(abstractor_abstraction, abstractor_abstraction_source, sentence[:sentence], sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil) - end + object_variants(abstractor_object_value, abstractor_object_value.abstractor_object_value_variants).each do |object_variant| + all_object_variants[object_variant] = abstractor_object_value + end + end + + all_object_variants.sort{|a,b| b[0].length <=> a[0].length}.each do |object_variant, abstractor_object_value| + match = parser.match_sentence(sentence[:sentence], Regexp.escape(object_variant)) + if match + scoped_sentence = Abstractor::NegationDetection.parse_negation_scope(sentence[:sentence]) + reject = ( + Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], predicate_variant) || + Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], predicate_variant) || + Abstractor::NegationDetection.negated_match_value?(scoped_sentence[:scoped_sentence], object_variant) || + Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], object_variant) + ) + if !reject + suggest(abstractor_abstraction, abstractor_abstraction_source, sentence[:sentence], sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], abstractor_object_value, nil, nil, nil, nil, range.begin, range.end) end end end @@ -303,15 +321,21 @@ def abstract_sentential_name_value(about, abstractor_abstraction, abstractor_abs end end - def suggest(abstractor_abstraction, abstractor_abstraction_source, match_value, sentence_match_value, source_id, source_type, source_method, section_name, suggested_value, unknown, not_applicable, custom_method, custom_explanation) + def suggest(abstractor_abstraction, abstractor_abstraction_source, match_value, sentence_match_value, source_id, source_type, source_method, section_name, suggested_value, unknown, not_applicable, custom_method, custom_explanation, begin_position, end_position) + # unless abstractor_abstraction.abstractor_suggestions.reload.select{|s| m = Regexp.new(Regexp.escape(object_variant.downcase)) =~ s.suggested_value}.any? + match_value.strip! unless match_value.nil? sentence_match_value.strip! unless sentence_match_value.nil? if abstractor_object_value?(suggested_value) abstractor_object_value = suggested_value suggested_value = suggested_value.value end - abstractor_suggestion = abstractor_abstraction.detect_abstractor_suggestion(suggested_value, unknown, not_applicable) - if !abstractor_suggestion + + abstractor_suggestion = abstractor_abstraction.detect_abstractor_suggestion(suggested_value, unknown, not_applicable) + abstractor_suggestion_partial = abstractor_abstraction.detect_abstractor_suggestion_partial(suggested_value, unknown, not_applicable, begin_position, end_position) + + abstractor_suggestion ||= abstractor_suggestion_partial + unless abstractor_suggestion abstractor_suggestion_status_needs_review = Abstractor::AbstractorSuggestionStatus.where(name: 'Needs review').first abstractor_suggestion = Abstractor::AbstractorSuggestion.create( abstractor_abstraction: abstractor_abstraction, @@ -326,7 +350,7 @@ def suggest(abstractor_abstraction, abstractor_abstraction_source, match_value, abstractor_suggestion_source = abstractor_suggestion.detect_abstractor_suggestion_source(abstractor_abstraction_source, sentence_match_value, source_id, source_type, source_method, section_name) if !abstractor_suggestion_source - Abstractor::AbstractorSuggestionSource.create( + abstractor_suggestion_source = Abstractor::AbstractorSuggestionSource.create( abstractor_abstraction_source: abstractor_abstraction_source, abstractor_suggestion: abstractor_suggestion, match_value: match_value, @@ -339,6 +363,11 @@ def suggest(abstractor_abstraction, abstractor_abstraction_source, match_value, custom_explanation: custom_explanation ) end + abstractor_suggestion_source_range = abstractor_suggestion_source.detect_abstractor_suggestion_source_range(begin_position, end_position) + if !abstractor_suggestion_source_range && begin_position && end_position + abstractor_suggestion_source.abstractor_suggestion_source_ranges.build(begin_position: begin_position, end_position: end_position) + abstractor_suggestion_source.save + end abstractor_suggestion end @@ -364,7 +393,7 @@ def create_unknown_abstractor_suggestion_name_only(about, abstractor_abstraction Abstractor::NegationDetection.manual_negated_match_value?(sentence[:sentence], predicate_variant) ) if !reject - suggest(abstractor_abstraction, abstractor_abstraction_source, predicate_variant.downcase, sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], nil, true, nil, nil, nil) + suggest(abstractor_abstraction, abstractor_abstraction_source, predicate_variant.downcase, sentence[:sentence], source[:source_id], source[:source_type].to_s, source[:source_method], source[:section_name], nil, true, nil, nil, nil, nil, nil) end end end @@ -378,7 +407,7 @@ def create_unknown_abstractor_suggestion(about, abstractor_abstraction, abstract #Create an 'unknown' suggestion based on matching nothing only if we have not made a suggstion abstractor_abstraction_source.normalize_from_method_to_sources(about).each do |source| if abstractor_abstraction.abstractor_suggestions(true).select { |abstractor_suggestion| abstractor_suggestion.unknown != true }.empty? - suggest(abstractor_abstraction, abstractor_abstraction_source, nil, nil, source[:source_id], source[:source_type].to_s, source[:source_method],source[:section_name], nil, true, nil, nil, nil) + suggest(abstractor_abstraction, abstractor_abstraction_source, nil, nil, source[:source_id], source[:source_type].to_s, source[:source_method],source[:section_name], nil, true, nil, nil, nil, nil, nil) end end end diff --git a/lib/abstractor/methods/models/abstractor_suggestion_source.rb b/lib/abstractor/methods/models/abstractor_suggestion_source.rb index a2b6567..c61bc6f 100644 --- a/lib/abstractor/methods/models/abstractor_suggestion_source.rb +++ b/lib/abstractor/methods/models/abstractor_suggestion_source.rb @@ -8,8 +8,16 @@ def self.included(base) # Associations base.send :belongs_to, :abstractor_abstraction_source base.send :belongs_to, :abstractor_suggestion + base.send :has_many, :abstractor_suggestion_source_ranges, dependent: :destroy # base.send :attr_accessible, :abstractor_abstraction_source, :abstractor_abstraction_source_id, :abstractor_suggestion, :abstractor_suggestion_id, :source_id, :source_type, :source_method, :match_value, :deleted_at, :sentence_match_value, :custom_method, :custom_explanation + base.send(:include, InstanceMethods) + end + + module InstanceMethods + def detect_abstractor_suggestion_source_range(begin_position, end_position) + abstractor_suggestion_source_ranges.where(begin_position: begin_position, end_position: end_position).first + end end end end diff --git a/lib/abstractor/methods/models/abstractor_suggestion_source_range.rb b/lib/abstractor/methods/models/abstractor_suggestion_source_range.rb new file mode 100644 index 0000000..6e5357b --- /dev/null +++ b/lib/abstractor/methods/models/abstractor_suggestion_source_range.rb @@ -0,0 +1,14 @@ +module Abstractor + module Methods + module Models + module AbstractorSuggestionSourceRange + def self.included(base) + base.send :include, SoftDelete + + # Associations + base.send :belongs_to, :abstractor_suggestion + end + end + end + end +end diff --git a/lib/abstractor/parser.rb b/lib/abstractor/parser.rb index b36700a..4138138 100644 --- a/lib/abstractor/parser.rb +++ b/lib/abstractor/parser.rb @@ -29,18 +29,24 @@ def initialize(abstractor_text, options = {}) end def scan(token, options = {}) - options[:word_boundary] = true if options[:word_boundary].nil? + options[:word_boundary] = true if options[:word_boundary].nil? + options[:match_data] = false if options[:match_data].nil? + regular_expression = prepare_token(token, options) at = prepare_abstractor_text if (regular_expression.nil? || at.nil?) [] + elsif options[:match_data] + # http://stackoverflow.com/questions/6804557/how-do-i-get-the-match-data-for-all-occurrences-of-a-ruby-regular-expression-in + at.to_enum(:scan,regular_expression).map{ Regexp.last_match } else at.scan(regular_expression) end end - def match(token) - regular_expression = prepare_token(token) + def match(token, options = {}) + options[:word_boundary] = true if options[:word_boundary].nil? + regular_expression = prepare_token(token, options) prepare_abstractor_text.match(regular_expression) unless regular_expression.nil? end diff --git a/spec/models/abstractor/abstractor_suggestion_source_range_spec.rb b/spec/models/abstractor/abstractor_suggestion_source_range_spec.rb new file mode 100644 index 0000000..b6af981 --- /dev/null +++ b/spec/models/abstractor/abstractor_suggestion_source_range_spec.rb @@ -0,0 +1,7 @@ +require 'spec_helper' + +module Abstractor + RSpec.describe AbstractorSuggestionSourceRange, :type => :model do + pending "add some examples to (or delete) #{__FILE__}" + end +end diff --git a/spec/models/abstractor/encounter_note_spec.rb b/spec/models/abstractor/encounter_note_spec.rb index b7cc6c3..817eb9a 100644 --- a/spec/models/abstractor/encounter_note_spec.rb +++ b/spec/models/abstractor/encounter_note_spec.rb @@ -163,6 +163,7 @@ expect(@encounter_note.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_kps).abstractor_suggestions.first.suggested_value).to eq('kps') end + #suggestions it "does not create another 'has_karnofsky_performance_status' abstraction suggestion upon re-abstraction (using the canonical name/value format)" do @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. KPS: 90.') @@ -227,6 +228,152 @@ expect(@encounter_note.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_kps).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '90% - Able to carry on normal activity; minor signs or symptoms of disease.'}.size).to eq(1) end + describe 'if match is a part of another matched value' do + before(:each) do + @h_y_abstractor_abstraction_schema = Abstractor::AbstractorAbstractionSchema.where( + predicate: 'has_hoehn_and_yahr_score', + display_name: 'Hoehn & Yahr stage', + abstractor_object_type_id: @list_object_type, + preferred_name: 'H&Y').first_or_create + + ['1', '2', '2.5', '3', '3.5', '4', '5'].each do |value| + abstractor_object_value = Abstractor::AbstractorObjectValue.where(value: value.to_s).first_or_create + Abstractor::AbstractorAbstractionSchemaObjectValue.where(abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema, abstractor_object_value: abstractor_object_value).first_or_create + end + + @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create + @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @name_value_rule) + end + + describe 'with name/value rule type' do + before(:each) do + @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create + @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @name_value_rule) + end + describe 'it does not create multiple name/value matches if match is a part of another overlapping matched value' do + it "using the canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + + it "using the squished canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + + it "using the sentential format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5.") + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + end + + describe 'it creates multiple name/value matches if matches do not overlap' do + it "using the canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5. H&Y: 2. H&Y: 5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + + it "using the squished canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5. H&Y2 or H&Y5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + + it "using the sentential format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5. On another hand H&Y can be 2 or 5.") + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + end + end + + describe 'with value rule type' do + before(:each) do + @abstractor_subject = Abstractor::AbstractorSubject.where( subject_type: 'EncounterNote', abstractor_abstraction_schema: @h_y_abstractor_abstraction_schema).first_or_create + @abstractor_abstraction_source = Abstractor::AbstractorAbstractionSource.create(abstractor_subject: @abstractor_subject, from_method: 'note_text', abstractor_abstraction_source_type: @source_type_nlp_suggestion, abstractor_rule_type: @value_rule) + end + describe 'it does not create multiple name/value matches if match is a part of another overlapping matched value' do + it "using the canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + + it "using the squished canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + + it "using the sentential format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5.") + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(0) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(0) + end + end + + describe 'it creates multiple name/value matches if matches do not overlap' do + it "using the canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y: 2.5. H&Y: 2. H&Y: 5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + + it "using the squished canonical name/value format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: 'The patient looks healthy. H&Y2.5. H&Y2 or H&Y5') + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + + it "using the sentential format" do + @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's H&Y is 2.5. On another hand H&Y can be 2 or 5.") + @encounter_note.abstract + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.size).to eq(3) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2.5'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '2'}.size).to eq(1) + expect(@encounter_note.reload.detect_abstractor_abstraction(@h_y_abstractor_abstraction_schema).abstractor_suggestions.select { |suggestion| suggestion.suggested_value == '5'}.size).to eq(1) + end + end + end + end + #custom suggestions it "creates one 'has_karnofsky_performance_status_date' abstraction suggestion (using a custom rule)", focus: false do @encounter_note = FactoryGirl.create(:encounter_note, note_text: "The patient looks healthy. The patient's kps is 90.") diff --git a/spec/models/abstractor/radiation_therapy_prescription_spec.rb b/spec/models/abstractor/radiation_therapy_prescription_spec.rb index a2b4b21..b020c05 100644 --- a/spec/models/abstractor/radiation_therapy_prescription_spec.rb +++ b/spec/models/abstractor/radiation_therapy_prescription_spec.rb @@ -103,7 +103,7 @@ radiation_therapy_prescription = FactoryGirl.create(:radiation_therapy_prescription, site_name: 'left parietal lobe and bilateral cerebral meninges') radiation_therapy_prescription.abstract - expect(Set.new(radiation_therapy_prescription.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_has_anatomical_location).abstractor_suggestions.map(&:suggested_value))).to eq(Set.new(['cerebral meninges', 'parietal lobe', 'meninges'])) + expect(Set.new(radiation_therapy_prescription.reload.detect_abstractor_abstraction(@abstractor_subject_abstraction_schema_has_anatomical_location).abstractor_suggestions.map(&:suggested_value))).to eq(Set.new(['cerebral meninges', 'parietal lobe'])) end it "creates one 'has_anatomical_location' abstraction suggestion given multiple identical matches" do