Skip to content

Commit

Permalink
Feat/jester import (#349)
Browse files Browse the repository at this point in the history
  • Loading branch information
@browjm4
browjm4 authored and GitHub Enterprise committed Apr 19, 2023
1 parent 0964bbc commit 526e548
Show file tree
Hide file tree
Showing 8 changed files with 176 additions and 38 deletions.
15 changes: 11 additions & 4 deletions AdminWebApp/src/api/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -659,8 +659,8 @@ export class Client {
return this.postRawReturn<ResultT<boolean>[]>(`/containers/${containerID}/import/datasources/${dataSourceID}/mappings/upgrade`, payload);
}

dataSourceJSONFileImport(containerID: string, dataSourceID: string, file: File): Promise<boolean> {
return this.postFile(`/containers/${containerID}/import/datasources/${dataSourceID}/imports`, 'import', file);
dataSourceJSONFileImport(containerID: string, dataSourceID: string, file: File, fastload?: boolean): Promise<boolean> {
return this.postFile(`/containers/${containerID}/import/datasources/${dataSourceID}/imports`, 'import', file, {fastLoad: fastload});
}

async uploadFile(containerID: string, dataSourceID: string, file: File): Promise<FileT> {
Expand Down Expand Up @@ -1565,7 +1565,7 @@ export class Client {
});
}

private async postFile(uri: string, inputName: string, file: File): Promise<boolean> {
private async postFile(uri: string, inputName: string, file: File, queryParams?: {[key: string]: any}): Promise<boolean> {
const config: AxiosRequestConfig = {};
config.headers = {'Access-Control-Allow-Origin': '*', 'Content-Type': 'multipart/form-data'};
config.validateStatus = () => {
Expand All @@ -1580,10 +1580,17 @@ export class Client {
config.auth = {username: this.config.username, password: this.config.password} as AxiosBasicCredentials;
}

let url: string;
if (queryParams) {
url = buildURL(this.config?.rootURL!, {path: uri, queryParams: queryParams!});
} else {
url = buildURL(this.config?.rootURL!, {path: uri})
}

const formData = new FormData();
formData.append(inputName, file);

const resp: AxiosResponse = await axios.post(buildURL(this.config?.rootURL!, {path: uri}), formData, config);
const resp: AxiosResponse = await axios.post(url, formData, config);

return new Promise<boolean>((resolve, reject) => {
if (resp.status < 200 || resp.status > 299) reject(resp.data.error);
Expand Down
2 changes: 2 additions & 0 deletions AdminWebApp/src/api/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ export type TimeseriesDataSourceConfig = {
value_nodes?: string[];
data_retention_days?: number;
raw_retention_enabled?: boolean;
fast_load_enabled?: boolean;
};

export type HttpDataSourceConfig = {
Expand Down Expand Up @@ -804,5 +805,6 @@ export function DefaultTimeseriesDataSourceConfig(): TimeseriesDataSourceConfig
kind: 'timeseries',
columns: [],
attachment_parameters: [],
fast_load_enabled: true,
};
}
18 changes: 9 additions & 9 deletions AdminWebApp/src/components/dataImport/importDataDialog.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,11 @@
<error-banner :message="errorMessage"></error-banner>
<v-row>
<v-col :cols="12">

<v-form
ref="form"
lazy-validation
>

<v-file-input label=".json, .xml, .csv" @change="addFiles"></v-file-input>

<v-form ref="form" lazy-validation>
<v-file-input
label=".json, .xml, .csv"
@change="addFiles"
/>
</v-form>
</v-col>
</v-row>
Expand Down Expand Up @@ -50,6 +47,9 @@
@Prop({required: false, default: false})
disabled!: boolean
@Prop({required: false, default: false})
fastload!: boolean
errorMessage = ""
loading = false
dialog = false
Expand All @@ -67,7 +67,7 @@
uploadImport() {
this.loading = true
if(this.filesToUpload) {
this.$client.dataSourceJSONFileImport(this.containerID, this.dataSourceID, this.filesToUpload)
this.$client.dataSourceJSONFileImport(this.containerID, this.dataSourceID, this.filesToUpload, this.fastload)
.then(() => {
this.dialog = false
this.errorMessage = ""
Expand Down
77 changes: 68 additions & 9 deletions AdminWebApp/src/components/dataSources/createDataSourceDialog.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,11 @@
</div>

<div v-if="newDataSource.adapter_type === 'timeseries'">
<v-checkbox
label="Enable Fastload Ingestion"
v-model="fastload"
/>

<p><b>{{$t('createDataSource.description')}}</b></p>
<p>{{$t('createDataSource.timeseriesDescription')}} <a href="https://github.com/idaholab/Deep-Lynx/wiki/Timeseries-Data-Sources" target="_blank">{{$t('dataMapping.here')}}.</a></p>

Expand Down Expand Up @@ -465,19 +470,23 @@

<v-col v-if="item.type === 'date'" :cols="12">
<v-text-field
:label="$t('dataMapping.dateFormatString')"
v-model="item.date_conversion_format_string"
:label="$t('dataMapping.dateFormatString')"
v-model="item.date_conversion_format_string"
:rules="[rules.dateString]"
>
<template slot="append-outer"><a href="https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE" target="_blank">{{$t('dataMapping.dateFormatStringHelp')}}</a></template>
<template slot="append-outer">
<a :href="timeseriesHelpLink()" target="_blank">
{{$t('dataMapping.dateFormatStringHelp')}}
</a>
</template>
</v-text-field>
</v-col>
<v-col v-if="item.type === 'date'" class="text-left">
<v-checkbox
:label="$t('dataMapping.isPrimaryTimestamp')"
v-model="item.is_primary_timestamp"
disabled
>
</v-checkbox>
:label="$t('dataMapping.isPrimaryTimestamp')"
v-model="item.is_primary_timestamp"
disabled
/>
</v-col>
</td>
</template>
Expand Down Expand Up @@ -632,6 +641,17 @@ export default class CreateDataSourceDialog extends Vue {
}
hideP6pass = true
authorized: string[] = []
fastload = true
defaultFormatString = '%Y-%m-%d %H:%M:%S.%f'
rules={
dateString: (value: any) => {
if (this.getFastload()) {
return value.includes('%') || 'Date String should be in strftime datetime format'
} else {
return !value.includes('%') || 'Date String should be in postgres datetime format'
}
}
}
newDataSource: DataSourceT = {
name: "",
Expand Down Expand Up @@ -668,6 +688,37 @@ export default class CreateDataSourceDialog extends Vue {
}
}
// for some reason calling the variable itself in the validation always results
// in the default "true". this getter is a workaround for that
getFastload() {
return this.fastload
}
@Watch('fastload')
dateStringChange() {
if (this.fastload) {
this.defaultFormatString = '%Y-%m-%d %H:%M:%S.%f'
} else {
this.defaultFormatString = 'YYYY-MM-DD HH24:MI:SS.US'
}
this.checkDateStrings()
}
checkDateStrings() {
this.timeseriesConfig.columns.forEach(col => {
if (col.date_conversion_format_string && col.date_conversion_format_string !== this.defaultFormatString) {
// if fastload enabled and no percent symbols, assume that string needs to be reformatted to rust format
// similarly, if fastload disabled but percent symbols present, reformat string to pg format
if (
(this.fastload && !col.date_conversion_format_string.includes('%'))
|| (!this.fastload && col.date_conversion_format_string.includes('%'))
) {
col.date_conversion_format_string = this.defaultFormatString
}
}
})
}
beforeMount() {
this.container = this.$store.getters.activeContainer;
}
Expand Down Expand Up @@ -712,6 +763,13 @@ export default class CreateDataSourceDialog extends Vue {
]
}
timeseriesHelpLink(): string {
if (this.fastload === true) {
return "https://docs.rs/chrono/0.4.24/chrono/format/strftime/index.html"
}
return "https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE"
}
selectAdapter(adapter: string) {
this.newDataSource.adapter_type = adapter
}
Expand Down Expand Up @@ -786,6 +844,7 @@ export default class CreateDataSourceDialog extends Vue {
}
case "timeseries": {
this.timeseriesConfig.fast_load_enabled = this.fastload;
this.newDataSource.config = this.timeseriesConfig;
this.newDataSource.active = true;
break;
Expand Down Expand Up @@ -895,7 +954,7 @@ export default class CreateDataSourceDialog extends Vue {
is_primary_timestamp: true,
unique: false,
type: 'date',
date_conversion_format_string: 'YYYY-MM-DD HH24:MI:SS.US',
date_conversion_format_string: this.fastload ? '%Y-%m-%d %H:%M:%S.%f' : 'YYYY-MM-DD HH24:MI:SS.US',
})
this.expandedTimeSeries.push(this.timeseriesConfig.columns[0])
Expand Down
70 changes: 61 additions & 9 deletions AdminWebApp/src/components/dataSources/editDataSourceDialog.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,11 @@
</div>

<div v-if="dataSource.adapter_type === 'timeseries'">
<v-checkbox
label="Enable Fastload Ingestion"
v-model="fastload"
/>

<h4>{{$t('dataMapping.tableDesign')}}<info-tooltip :message="$t('dataMapping.tableDesignHelp')"></info-tooltip></h4>

<v-data-table
Expand Down Expand Up @@ -433,19 +438,23 @@

<v-col v-if="item.type === 'date'" :cols="12">
<v-text-field
:label="$t('dataMapping.dateFormatString')"
v-model="item.date_conversion_format_string"
:label="$t('dataMapping.dateFormatString')"
v-model="item.date_conversion_format_string"
:rules="[rules.dateString]"
>
<template slot="append-outer"><a href="https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE" target="_blank">{{$t('dataMapping.dateFormatStringHelp')}}</a></template>
<template slot="append-outer">
<a :href="timeseriesHelpLink()" target="_blank">
{{$t('dataMapping.dateFormatStringHelp')}}
</a>
</template>
</v-text-field>
</v-col>
<v-col v-if="item.type === 'date'" class="text-left">
<v-checkbox
:label="$t('dataMapping.isPrimaryTimestamp')"
v-model="item.is_primary_timestamp"
disabled
>
</v-checkbox>
:label="$t('dataMapping.isPrimaryTimestamp')"
v-model="item.is_primary_timestamp"
disabled
/>
</v-col>
</td>
</template>
Expand Down Expand Up @@ -542,7 +551,7 @@
</template>

<script lang="ts">
import {Component, Prop, Vue} from "vue-property-decorator"
import {Component, Prop, Vue, Watch} from "vue-property-decorator"
import {
DataSourceT,
HttpDataSourceConfig, TimeseriesDataSourceConfig,
Expand All @@ -567,6 +576,16 @@ export default class EditDataSourceDialog extends Vue {
authMethods = [{text: "Basic", value: 'basic'}, {text: "Token", value: 'token'}]
expandedTimeSeries: any[] = []
hideP6pass = true
fastload = (this.dataSource.config as TimeseriesDataSourceConfig).fast_load_enabled
rules={
dateString: (value: any) => {
if (this.getFastload()) {
return value.includes('%') || 'Date String should be in strftime datetime format'
} else {
return !value.includes('%') || 'Date String should be in postgres datetime format'
}
}
}
dataTypes = [
'number',
Expand All @@ -589,6 +608,28 @@ export default class EditDataSourceDialog extends Vue {
]
}
// for some reason calling the variable itself in the validation always results
// in the default "true". this getter is a workaround for that
getFastload() {
return this.fastload
}
@Watch('fastload')
validateDateString() {
// @ts-ignore
this.$refs.form!.validate()
}
// upon any changes to the timeseries config, re-validate the form
// to ensure date string format passes validation
@Watch('dataSource.config', {deep: true})
checkDateString() {
if (this.dataSource.adapter_type === 'timeseries' && (this.dataSource.config as TimeseriesDataSourceConfig).columns.length > 0) {
// @ts-ignore
this.$refs.form!.validate()
}
}
timeSeriesHeader() {
return [
{
Expand Down Expand Up @@ -629,9 +670,20 @@ export default class EditDataSourceDialog extends Vue {
}
}
timeseriesHelpLink(): string {
if (this.fastload) {
return "https://docs.rs/chrono/0.4.24/chrono/format/strftime/index.html"
}
return "https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE"
}
updateDataSource() {
this.dataSource.config!.data_retention_days = parseInt(String(this.dataSource.config!.data_retention_days), 10)
if (this.dataSource.adapter_type === 'timeseries') {
(this.dataSource.config as TimeseriesDataSourceConfig).fast_load_enabled = this.fastload;
}
this.$client.updateDataSource(this.containerID, this.dataSource)
.then((dataSource)=> {
this.$emit("dataSourceUpdated", dataSource)
Expand Down
5 changes: 3 additions & 2 deletions AdminWebApp/src/views/DataImports.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@
<import-data-dialog
:dataSourceID="selectedDataSource.id"
:containerID="containerID"
:fastload="selectedDataSource.config.fast_load_enabled"
:disabled="!selectedDataSource.active || selectedDataSource.archived"
@importUploaded="listImports">
</import-data-dialog>
@importUploaded="listImports"
/>
</div>
</v-col>
</template>
Expand Down
Loading

0 comments on commit 526e548

Please sign in to comment.