Fix/merge (#670)

.docker-env

@@ -1,8 +1,7 @@
 # should be in the format postgresql://user:password@hostname:port/database_name
 # :port is optional and if included will usually be :5432
-CORE_DB_CONNECTION_STRING=postgresql://postgres:root@timescaledb/deep_lynx_dev
-TIMESCALEDB_ENABLED=true
-WAIT_HOSTS=rabbitmq:5672
+CORE_DB_CONNECTION_STRING=postgresql://postgres:root@postgres/deep_lynx_dev
+TIMESCALEDB_ENABLED=false
 
 # valid options are blank (defaults to memory), memory, or redis
 CACHE_PROVIDER=redis
@@ -21,11 +20,6 @@ ENCRYPTION_KEY_PATH=
 # plaintext secret used to generate secure session markers
 SESSION_SECRET=CHANGEME
 
-# controls which queue system to use, possible values are database, rabbitmq and azure_service_bus
-QUEUE_SYSTEM=rabbitmq
-# RabbitMQ connection string
-RABBITMQ_URL=amqp://deeplynx:root@rabbitmq:5672
-
 # controls whether or not DeepLynx should emit data events
 EMIT_EVENTS=true
 
@@ -92,4 +86,4 @@ HPC_EMAIL=
 
 # Redirect for adapter authorizations
 # P6
-P6_REDIRECT_ADDRESS=http://localhost:8181
+P6_REDIRECT_ADDRESS=http://localhost:8181

.dockerignore

@@ -1,7 +1,7 @@
-server/legacy/.env
-server/legacy/.env-sample
-server/legacy/node_modules
+server/.env
+server/.env-sample
+server/node_modules
 npm-debug.log
-server/legacy/dist
+server/dist
 ui/AdminWebApp/node_modules
 ui/WebGLViewer/node_modules

.gitignore

@@ -17,24 +17,24 @@ profile-logs
 /src/http_server/authentication/saml/localhost.cert
 /src/http_server/authentication/saml/localhost.xml
 /tools/ontology_extractor/diamond.owl
-/private-key.pem
+**/private-key.pem
 /private-key.key
-/public-key.pem
-/server/legacy/src/tests/test-data.json
-/server/legacy/src/tests/test-data.csv
-/server/legacy/src/tests/*.csv
-/server/legacy/src/tests/private-key.pem
-/server/legacy/src/tests/public-key.pem
-/server/legacy/src/tests/*_export.json
-/server/legacy/src/tests/mochawesome-report
-/server/legacy/src/tests/coverage
+**/public-key.pem
+/server/src/tests/test-data.json
+/server/src/tests/test-data.csv
+/server/src/tests/*.csv
+/server/src/tests/private-key.pem
+/server/src/tests/public-key.pem
+/server/src/tests/*_export.json
+/server/src/tests/mochawesome-report
+/server/src/tests/coverage
 /**/test-timeseries-datasource-graphql.json
 /NodeLibraries/dl-fast-load/target/
 /NodeLibraries/dl-fast-load/index.node
 /RedisLoader/target/
 /RedisLoader/RedisLoader.iml
-/AdminWebApp/src/**/*.js
-/AdminWebApp/src/**/*.js.map
+*/AdminWebApp/src/**/*.js
+*/AdminWebApp/src/**/*.js.map
 /NodeLibraries/dl-fast-load/build-output.txt
 /NodeLibraries/deeplynx-timeseries/
 /NodeLibraries/redis-graph-loader/
@@ -56,21 +56,20 @@ devenv.local.nix
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
-/public-key.pem
+**/public-key.pem
 /yarn-error.log
 /.yarn/
-/server/legacy/public-key.pem
-/server/legacy/private-key.pem
-/server/deeplynx/target/
-/server/legacy/.yarn/
+/server/public-key.pem
+/server/private-key.pem
+/server/.yarn/
 /CAINLROOT.cer
-# ignore artifacts of rust modules both pre- and post- switch- 
+# ignore artifacts of rust modules both pre- and post- switch-
 # this will make branch switching easier for existing branches
 # during the transition period
-/server/legacy/NodeLibraries/deeplynx/Cargo.lock
-/server/legacy/NodeLibraries/deeplynx/target
-/server/legacy/NodeLibraries/deeplynx/*.node
+/server/NodeLibraries/deeplynx/Cargo.lock
+/server/NodeLibraries/deeplynx/target
+/server/NodeLibraries/deeplynx/*.node
 
-/NodeLibraries/deeplynx/Cargo.lock
-/NodeLibraries/deeplynx/target
-/NodeLibraries/deeplynx/*.node
+**/NodeLibraries/deeplynx/Cargo.lock
+**/NodeLibraries/deeplynx/target
+**/NodeLibraries/deeplynx/*.node

Dockerfile

@@ -17,7 +17,7 @@ ENV VUE_APP_DEEP_LYNX_APP_ID="root"
 # turn off jobs on the main thread as this spins up PM2 with the worker
 ENV RUN_JOBS=false
 # set the default db to the one we'd see in the docker compose
-ENV CORE_DB_CONNECTION_STRING=postgresql://postgres:root@timescaledb:5432/deep_lynx_dev
+ENV CORE_DB_CONNECTION_STRING=postgresql://postgres:root@postgres:5432/deep_lynx_dev
 
 RUN apk update
 RUN apk add build-base musl-dev openssl-dev
@@ -36,33 +36,27 @@ WORKDIR /srv/deeplynx
 COPY . .
 
 # triple check we're not pulling in node_modules from the host system
-RUN rm -rf /srv/deeplynx/server/legacy/node_modules
+RUN rm -rf /srv/deeplynx/server/node_modules
 RUN rm -rf /srv/deeplynx/ui/AdminWebApp/node_modules
 RUN rm -rf /srv/deeplynx/ui/WebGLViewer/node_modules
 
-WORKDIR /srv/deeplynx/server/deeplynx
-RUN cargo install --path .
+WORKDIR /srv/deeplynx/server
+RUN yarn install;
+RUN yarn run build;
 
 FROM node:alpine3.19 as production
 ENV DEVELOPMENT_MODE=false
 
 RUN apk update && apk add supervisor
-RUN mkdir -p /srv/deeplynx/server/legacy
+RUN mkdir -p /srv/deeplynx/server
 
 # need pm2 to run legacy server
 RUN npm install npm@latest --location=global
 RUN npm update --location=global
 RUN npm install pm2 --location=global
 
-COPY --from=build /srv/deeplynx/server/legacy /srv/deeplynx/server/legacy
-COPY --from=build /usr/local/cargo/bin/deeplynx /usr/local/bin/deeplynx
-COPY --from=build /srv/deeplynx/server/deeplynx/configs /configs
+COPY --from=build /srv/deeplynx/server /srv/deeplynx/server
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
-# Add docker-compose-wait tool ----------------------
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait
-RUN chmod +x /wait
-
 EXPOSE 8090
-EXPOSE 4000
-CMD /wait && /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+CMD /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf

docker-compose.yml

@@ -4,28 +4,20 @@ services:
     image: "idaholab/deeplynx:latest"
     restart: on-failure
     depends_on:
-      - timescaledb
-      - rabbitmq
+      - postgres
     ports:
       - "8090:8090"
     env_file:
       - "./.docker-env"
-  timescaledb:
+  postgres:
     environment:
       - POSTGRES_PASSWORD=root
-    image: "timescale/timescaledb:latest-pg12"
+      - POSTGRES_DB=deep_lynx_dev
+    image: "library/postgres:16.3"
     ports:
       - "5432:5432"
   redis:
     image: "redis/redis-stack:6.2.6-v9"
     ports:
       - "6379:6379"
       - "8001:8001"
-  rabbitmq:
-    environment:
-      - RABBITMQ_DEFAULT_USER=deeplynx
-      - RABBITMQ_DEFAULT_PASS=root
-    image: "rabbitmq"
-    ports:
-      - "5672:5672"
-      - "15672:15672"

readme.md

@@ -58,38 +58,36 @@ These instructions build out the legacy node.js backend and the UI for DeepLynx.
 
 - **Required** - PostgreSQL ^12.x
 - **Required** - `pg-crypto` Postgres extension (automatically included with Postgres > 12 and in the Docker images)
-- [TimescaleDB Postgres Extension](https://www.timescale.com/) - needed for raw data retention and time-series data
 
 You must follow these steps in the exact order given. Failure to do so will cause DeepLynx to either fail to launch, or launch with problems.
 
 1. NodeJS must be installed. You can find the download for your platform here: https://nodejs.org/en/download/ **note** - Newer versions of Node may be incompatible with some of the following commands. The most recent version tested that works fully is 16.13.0 - the latest LTS version.  
 
 2. Clone the DeepLynx [repository](https://github.inl.gov/Digital-Engineering/DeepLynx/tree/main).
 
-3. Navigate to `server/legacy`.
+3. Navigate to the `server` directory.
 
 4. Run `yarn install` to set up all the node library dependencies.
 
 5. Copy and rename `.env-sample` to `.env`.
 
-6. Update `.env` file. See the `readme` or comments in the file itself for details. The main setting people usually change is setting `TIMESCALEDB_ENABLED=true` if they plan on ever working with time-series data.
+6. Update `.env` file. See the `readme` or comments in the file itself for details.
 
-7. To build the database using docker, follow step **a**. To use a dedicated PostgreSQL database, follow step **b**. Then continue to step 7.   
+7. To build a dockerized PostgreSQL database, follow step **a**. To use a native PostgreSQL database, follow step **b**. Then continue to step 8.   
 
 - 7a) Building the database using Docker:  
      - Ensure Docker is installed. You can find the download here: https://www.docker.com/products/docker-desktop.  
      - Run `npm run docker:postgres:build` to create a docker image containing a Postgres data source.  
      - Mac users may need to create the directory to mount to the docker container at `/private/var/lib/docker/basedata`. If this directory does not exist, please create it (you may need to use `sudo` as in `sudo mkdir /private/var/lib/docker/basedata`).  
      - Verify that image is properly created. See the screenshot below from Docker Desktop.
 ![image](uploads/e1d906d0399b1e4f890bf61035e5b64c/image.png)
-     - Run `npm run docker:postgres:run` to run the created docker image (For Mac users, there is an alternative command `npm run mac:docker:postgres:run`).  
-     - **Alternatively** you may use `npm run docker:timescale:run` (`npm run mac:docker:timescale:run` for Mac)to run a Postgres Docker image with the TimescaleDB extension already installed - to use TimescaleDB change the `.env` environment variable `TIMESCALEDB_ENABLED` to be `true`
+     - Run `npm run docker:postgres:run` to run the created docker image (For Mac users, there is an alternative command `npm run mac:docker:postgres:run`).
 - 7b) Building the database using a dedicated PostgreSQL database:  
      - Ensure PostgreSQL is installed. You can find the download here: https://www.postgresql.org/download/. Please see [this page](DeepLynx-Requirements) for the latest requirements on PostgreSQL version.  
      - Run pgAdmin and create a new database. The database name should match whatever value is provided in the `CORE_DB_CONNECTION_STRING` of the `.env` file. The default value is `deep_lynx`.  
      - Ensure a user has been created that also matches the `CORE_DB_CONNECTION_STRING` and that the user's password has been set appropriately. The default username is `postgres` and the default password is `deeplynxcore`.  
 
-8. Run `yarn run build` to build the internal modules and bundled administration GUI. **Note** You must re-run this command  if you make changes to the administration GUI.
+8. Run `yarn run build` to build the internal modules and bundled administration GUI. **Note** You must re-run this command if you make changes to the administration GUI.
 
 * NOTE: If you are on some sort of encrypted network, you may encounter an error similar to the following when attempting to set up any rust libraries: `warning: spurious network error... SSL connect error... The revocation function was unable to check revocation for the certificate.` This can be solved by navigating to your root cargo config file (`~/.cargo/config.toml`) file and adding the following lines. If you do not have an existing config.toml file at your root `.cargo` directory, you will need to make one:
 
@@ -106,24 +104,13 @@ check-revoke = false
 
 **Note:** DeepLynx ships with a Vue single page application which serves as the primary UI for the DeepLynx system. You can run this [separately](Administration-Web-App-Installation) (and it's recommended to do so if you're developing it).
 
-**The bundled admin web GUI can be accessed at `{{your base URL}}` - default is `localhost:8090`**
-
-### **Enabling TimescaleDB**
-
-DeepLynx ships with the capability to utilize a Postgres plugin called TimescaleDB. We use this for the storage of time-series data as well as a potential target for raw data retention. This is a powerful tool and you must have it enabled in order to store time-series data on nodes.
-
-1. Change the `TIMESCALEDB_ENABLED` environment variable to read `true`
-2. Restart the application.
-
-**Note:** Once you enable TimescaleDB you **cannot** disable it. Please make sure you absolutely need this extension of DeepLynx before taking steps to enable.
-
 **The bundled admin web GUI can be accessed at `{{your base URL}}` - default is `localhost:8090`**
 
 ### **Configuration**
 
 This application's configuration relies on environment variables of its host system. It is best to rely on your CI/CD pipeline to inject those variables into your runtime environment.
 
-In order to facilitate local development, a method has been provided to configure the application as if you were setting environment variables on your local machine. Including a `.env` file at the projects root and using the `npm run watch`, `npm run start`, or any of the `npm run docker:*` commands will start the application loading the listed variables in that file. See the `.env-sample` file included as part of the project for a list of required variables and formatting help.
+In order to facilitate local development, a method has been provided to configure the application as if you were setting environment variables on your local machine. Including a `.env` file at the projects root and using the `yarn run watch`, `yarn run start`, or any of the `yarn run docker:*` commands will start the application loading the listed variables in that file. See the `.env-sample` file included as part of the project for a list of required variables and formatting help.
 
 ### **Database Migrations**
 
@@ -143,7 +130,6 @@ Below is a list of all `yarn run` commands as listed in the `package.json` file.
 - `docker:postgres:build` Creates a Docker image containing a Postgres 12 data source.
 - `docker:postgres:run` Runs previously created Postgres image.
 - `docker:postgres:clean` Stops the Postgres Docker container run by the command above and deletes the container and image.
-- `docker:timescale:run` Runs a Postgres 12 Docker container with TimescaleDB already installed.
 - `build` Compiles the application
 - `start` Runs the compiled application 
 - `watch` Starts the application and rebuilds it each time you make a change to the code. **Note:** this command will not rebuild the bundled Admin Web Application

server/legacy/.env-sample → server/.env-sample

@@ -72,6 +72,9 @@ MINIO_SSL=false
 MINIO_ACCESS_KEY=
 MINIO_SECRET_KEY=
 
+# determines wheter or not this instance of DL is the main instance or a supporting node in a cluster
+IS_NODE=false
+
 # controls how many records a data source can insert into the database in a single transaction
 DATA_SOURCE_RECEIVE_BUFFER=1000
 # controls how often the data source job is ran

server/NodeLibraries/deeplynx/Cargo.toml

@@ -0,0 +1,46 @@
+[package]
+edition = "2021"
+name = "deeplynx"
+version = "0.0.1"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
+napi = { version = "2.16.6", default-features = false, features = ["napi4", "chrono", "tokio_rt", "async", "chrono_date", "serde", "serde_json", "tokio_io_util", "tokio_full"] }
+napi-derive = "2.16.4"
+sqlx = { version = "^0.7.4", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid", "macros", "bigdecimal"] }
+serde = { version = "^1.0.202", features = ["derive"] }
+serde_json = "1.0.117"
+tokio = { version = "1.37.0", features = ["full", "io-util"] }
+thiserror = "1.0.61"
+redis = { version = "0.25.3", features = ["tokio-comp", "tokio-rustls-comp"] }
+serde_yaml = "0.9.25"
+csv-async = { version = "1.3.0", features = ["tokio", "tokio-stream"] }
+futures = { version = "0.3.30", features = [] }
+futures-util = "0.3.30"
+tokio-util = { version = "0.7.11", features = ['compat', "io-util", "io"] }
+serial_test = "3.1.1"
+indexmap = "2.2.6"
+include_dir = "0.7.3"
+chrono = { version = "0.4.38", features = ["serde"] }
+uuid = { version = "1.8.0", features = ["serde", "fast-rng", "v4"] }
+validator = { version = "0.18.1", features = ["derive"] }
+rand = "0.9.0-alpha.1"
+bytes = "1.6.0"
+csv = "1.3.0"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+proc-macro2 = "1.0.83"
+sha2 = "0.9.9"
+base64 = { version = "0.22.1", features = ["default", "std"] }
+polars = { version = "0.40.0", features = ["strings", "performant", "lazy", "sql", "streaming", "temporal", "object", "cse", "serde", "serde-lazy", "dynamic_group_by", "partition_by", "rows", "is_between", "list_to_struct", "list_eval", "cov"] }
+polars-arrow = { version = "0.40.0", features = ["arrow_rs"] }
+arrow = { version = "51.0.0" }
+
+[build-dependencies]
+napi-build = "2.1.3"
+
+[profile.release]
+lto = true

server/legacy/NodeLibraries/deeplynx/index.d.ts → server/NodeLibraries/deeplynx/index.d.ts

@@ -63,3 +63,22 @@ export class RedisGraphLoader {
   init(config: Configuration): Promise<void>
   generateRedisGraph(containerId: string, timestamp?: string | undefined | null, ttl?: number | undefined | null): Promise<string>
 }
+export type JsSnapshotGenerator = SnapshotGenerator
+export class SnapshotGenerator {
+  constructor()
+  /**
+   * # Safety
+   *
+   * This function should be called before any work done on the object
+   * This generates the node snapshot dataframe and stores it on the SnapshotGenerator instance. This
+   * MUST be run before you attempt to find any nodes.
+   */
+  init(config: Configuration, containerId: string, timestamp?: string | undefined | null): Promise<void>
+  /**
+   * Find all the nodes that match a given set of parameters. Parameters must be EdgeParameters passed
+   * in as JSON in order to handle the fact that the value could be any valid JSON data-type. This function
+   * returns only the _database_ ids of the matching nodes - this is in order to avoid expensive serialization
+   * across the border.
+   */
+  findNodes(parametersJson: string): Promise<Array<string>>
+}