From 3a0c517c8e8c5c9d5ac2e7c7d9c34a460bdf05c3 Mon Sep 17 00:00:00 2001 From: remonatef94 Date: Tue, 29 Aug 2023 13:44:57 +0300 Subject: [PATCH 1/3] removing_slots_without_replacement --- src/main.ts | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index 7ea3ba8..a4d9775 100644 --- a/src/main.ts +++ b/src/main.ts @@ -632,7 +632,8 @@ export const datasetFromAST = async ( importHandler?: IFileImporter, currPath?: string ) => { - const operatorDefinitions = definitionsFromAST(initialAst, importHandler, currPath); + let operatorDefinitions = definitionsFromAST(initialAst, importHandler, currPath); + const operatorDefinitionsOriginal = JSON.parse(JSON.stringify(operatorDefinitions)); if (!operatorDefinitions) { return; } @@ -709,6 +710,49 @@ export const datasetFromAST = async ( false, globalCache ); + + // extract list of slots from selected sentence + type selectedSentenceSlotsDictType = {[key: string]: [string];}; + const selectedSentenceSlots: selectedSentenceSlotsDictType = {}; + for (const key in intentSentence) { + if (intentSentence.hasOwnProperty(key)) { + const sentencePart = intentSentence[key]; + if (sentencePart['type'] == "Slot"){ + const slotKey = String(sentencePart['slot']) + if (!selectedSentenceSlots.hasOwnProperty(slotKey)) { + selectedSentenceSlots[slotKey] = [sentencePart['value']]; + } else { + selectedSentenceSlots[slotKey].push(sentencePart['value']); + } + } + } + } + + // filter operatorDefinitions from selected intent slots + for (const slotName in selectedSentenceSlots){ + const slotValues = selectedSentenceSlots[slotName] + const slotSentences = operatorDefinitions.Slot[slotName].inner + for (const slotValue of slotValues){ + const targetSlotSentence = slotValue + // loop on slot sentences to pop from it if matched with target sentence + slotSentences.forEach((sentence, index) => { + const sentenceText = sentence['sentence'][0]['value']; + if (sentenceText == targetSlotSentence){ + slotSentences.splice(index, 1); + } + }); + } + operatorDefinitions.Slot[slotName].inner = slotSentences + } + + // Restore operatorDefinitions Slots if no slots available in any of slots + for (const slotName in operatorDefinitions.Slot){ + const slotValues = operatorDefinitions.Slot[slotName] + if (slotValues.inner.length < 1){ + operatorDefinitions.Slot[slotName].inner = operatorDefinitionsOriginal.Slot[slotName].inner.slice(); + } + } + const utterance = chatitoFormatPostProcess(intentSentence); const utteranceString = utterance.reduce((p, n) => p + n.value, ''); if (!collitionsCache[utteranceString]) { From 39f5c8d7bf754ad161be71da31ef48214d5b1840 Mon Sep 17 00:00:00 2001 From: remonatef94 Date: Wed, 30 Aug 2023 15:42:13 +0300 Subject: [PATCH 2/3] Apply remove without replacement logic on EntityVariationExtraction level --- src/main.ts | 57 +++++++++++++---------------------------------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/src/main.ts b/src/main.ts index a4d9775..a698c29 100644 --- a/src/main.ts +++ b/src/main.ts @@ -202,6 +202,18 @@ export const getVariationsFromEntity = async ( return []; } const sentence = ed.inner[sentenceIndex].sentence; + // Apply removing slots variation without replacement + if (ed.type == "SlotDefinition" && ed['variation'] !== null){ + const slotKeyInEntities = ed.key + "#" + ed.variation + // if one sentences left in the slot , then refill + if (entities.Slot[slotKeyInEntities].inner.length < 2){ + entities.Slot[slotKeyInEntities].inner = operatorDefinitionsOriginal.Slot[slotKeyInEntities].inner.slice() + } + else{ + entities.Slot[slotKeyInEntities].inner.splice(sentenceIndex, 1) + } + } + let accumulator: ISentenceTokens[] = []; // For slots where a sentence is composed of only one alias, we add the synonym tag, // to denote that the generated alias is a synonym of its alias name @@ -626,6 +638,7 @@ export const definitionsFromAST = (initialAst: IChatitoEntityAST[], importHandle return operatorDefinitions; }; +let operatorDefinitionsOriginal: {[key: string]: any;} = {} export const datasetFromAST = async ( initialAst: IChatitoEntityAST[], writterFn: IUtteranceWriter, @@ -633,7 +646,7 @@ export const datasetFromAST = async ( currPath?: string ) => { let operatorDefinitions = definitionsFromAST(initialAst, importHandler, currPath); - const operatorDefinitionsOriginal = JSON.parse(JSON.stringify(operatorDefinitions)); + operatorDefinitionsOriginal = JSON.parse(JSON.stringify(operatorDefinitions)); if (!operatorDefinitions) { return; } @@ -711,48 +724,6 @@ export const datasetFromAST = async ( globalCache ); - // extract list of slots from selected sentence - type selectedSentenceSlotsDictType = {[key: string]: [string];}; - const selectedSentenceSlots: selectedSentenceSlotsDictType = {}; - for (const key in intentSentence) { - if (intentSentence.hasOwnProperty(key)) { - const sentencePart = intentSentence[key]; - if (sentencePart['type'] == "Slot"){ - const slotKey = String(sentencePart['slot']) - if (!selectedSentenceSlots.hasOwnProperty(slotKey)) { - selectedSentenceSlots[slotKey] = [sentencePart['value']]; - } else { - selectedSentenceSlots[slotKey].push(sentencePart['value']); - } - } - } - } - - // filter operatorDefinitions from selected intent slots - for (const slotName in selectedSentenceSlots){ - const slotValues = selectedSentenceSlots[slotName] - const slotSentences = operatorDefinitions.Slot[slotName].inner - for (const slotValue of slotValues){ - const targetSlotSentence = slotValue - // loop on slot sentences to pop from it if matched with target sentence - slotSentences.forEach((sentence, index) => { - const sentenceText = sentence['sentence'][0]['value']; - if (sentenceText == targetSlotSentence){ - slotSentences.splice(index, 1); - } - }); - } - operatorDefinitions.Slot[slotName].inner = slotSentences - } - - // Restore operatorDefinitions Slots if no slots available in any of slots - for (const slotName in operatorDefinitions.Slot){ - const slotValues = operatorDefinitions.Slot[slotName] - if (slotValues.inner.length < 1){ - operatorDefinitions.Slot[slotName].inner = operatorDefinitionsOriginal.Slot[slotName].inner.slice(); - } - } - const utterance = chatitoFormatPostProcess(intentSentence); const utteranceString = utterance.reduce((p, n) => p + n.value, ''); if (!collitionsCache[utteranceString]) { From edf735ec3892ff45e990bf281e1194f3ef49dc07 Mon Sep 17 00:00:00 2001 From: remonatef94 Date: Sun, 3 Sep 2023 14:55:34 +0300 Subject: [PATCH 3/3] new-release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7d887b8..f48eae7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "chatito", - "version": "2.4.0", + "version": "2.4.1", "description": "Generate training datasets for NLU chatbots using a simple DSL", "bin": { "chatito": "./dist/bin.js"