From e13257c33a2927691ea3ca3e59aa0b2a2300e715 Mon Sep 17 00:00:00 2001 From: "Dr. Daniel Hauschildt" <14251188+DanielHauschildt@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:02:59 +0100 Subject: [PATCH] fix (web): Updated Onnx runtime to make it work for new chrome version. (#140) --- package-lock.json | 27 ++++++----- package.json | 2 +- packages/node-e2e/package.json | 2 +- packages/node-examples/package.json | 2 +- packages/node/package.json | 2 +- packages/web-data/.resources.mjs | 21 ++++++++- ...0-Added_ThirdPartyLicenses_json_Added.yaml | 9 ++++ packages/web-data/package.json | 4 +- .../vite-project/.vercel/README.txt | 11 +++++ .../vite-project/.vercel/project.json | 4 ++ .../vite-project/package-lock.json | 18 ++++---- .../web-examples/vite-project/package.json | 2 +- packages/web/CHANGELOG.md | 6 +++ ...0-Added_ThirdPartyLicenses_json_Added.yaml | 9 ++++ packages/web/package.json | 10 ++--- packages/web/src/onnx.ts | 37 ++++++--------- packages/web/src/resource.ts | 45 +++++++++++++++++-- scripts/package-resources.mjs | 15 ++++--- 18 files changed, 161 insertions(+), 65 deletions(-) create mode 100644 packages/web-data/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml create mode 100644 packages/web-examples/vite-project/.vercel/README.txt create mode 100644 packages/web-examples/vite-project/.vercel/project.json create mode 100644 packages/web/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml diff --git a/package-lock.json b/package-lock.json index 82c2615..7c80074 100644 --- a/package-lock.json +++ b/package-lock.json @@ -655,6 +655,7 @@ "version": "4.17.12", "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", + "dev": true, "dependencies": { "@types/lodash": "*" } @@ -3593,18 +3594,23 @@ } }, "node_modules/onnxruntime-web": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.18.0.tgz", - "integrity": "sha512-o1UKj4ABIj1gmG7ae0RKJ3/GT+3yoF0RRpfDfeoe0huzRW4FDRLfbkDETmdFAvnJEXuYDE0YT+hhkia0352StQ==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.1.tgz", + "integrity": "sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==", "dependencies": { "flatbuffers": "^1.12.0", "guid-typescript": "^1.0.9", "long": "^5.2.3", - "onnxruntime-common": "1.18.0", + "onnxruntime-common": "1.20.1", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.1.tgz", + "integrity": "sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==" + }, "node_modules/os-browserify": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", @@ -5796,15 +5802,15 @@ "version": "1.5.5", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { - "@types/lodash-es": "^4.17.12", - "@types/ndarray": "~1.0.14", - "@types/node": "~20.3.0", "lodash-es": "^4.17.21", "ndarray": "~1.0.0", - "onnxruntime-web": "~1.18.0", + "onnxruntime-web": "~1.20.1", "zod": "^3.23.8" }, "devDependencies": { + "@types/lodash-es": "^4.17.12", + "@types/ndarray": "~1.0.14", + "@types/node": "~20.3.0", "assert": "~2.0.0", "esbuild": "~0.18.0", "glob": "~10.3.0", @@ -5823,7 +5829,7 @@ "version": "1.5.5", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { - "onnxruntime-web": "~1.18.0" + "onnxruntime-web": "~1.20.1" } }, "packages/web/node_modules/@esbuild/android-arm": { @@ -6181,7 +6187,8 @@ "packages/web/node_modules/@types/node": { "version": "20.3.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.3.tgz", - "integrity": "sha512-wheIYdr4NYML61AjC8MKj/2jrR/kDQri/CIpVoZwldwhnIrD/j9jIU5bJ8yBKuB2VhpFV7Ab6G2XkBjv9r9Zzw==" + "integrity": "sha512-wheIYdr4NYML61AjC8MKj/2jrR/kDQri/CIpVoZwldwhnIrD/j9jIU5bJ8yBKuB2VhpFV7Ab6G2XkBjv9r9Zzw==", + "dev": true }, "packages/web/node_modules/ansi-styles": { "version": "4.3.0", diff --git a/package.json b/package.json index 8e3cf45..ada5b13 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "1.5.5", + "version": "1.5.6", "name": "workspace", "workspaces": [ "packages/web", diff --git a/packages/node-e2e/package.json b/packages/node-e2e/package.json index fca8f67..093dd01 100644 --- a/packages/node-e2e/package.json +++ b/packages/node-e2e/package.json @@ -6,5 +6,5 @@ "@imgly/background-removal-node": "file:../node", "jest": "^29.6.2" }, - "version": "1.5.5" + "version": "1.5.6" } diff --git a/packages/node-examples/package.json b/packages/node-examples/package.json index 85c8f36..a4c09f0 100644 --- a/packages/node-examples/package.json +++ b/packages/node-examples/package.json @@ -9,5 +9,5 @@ "@imgly/background-removal-node": "file:../node", "uuidv4": "^6.2.13" }, - "version": "1.5.5" + "version": "1.5.6" } diff --git a/packages/node/package.json b/packages/node/package.json index f24f840..9c61a72 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@imgly/background-removal-node", - "version": "1.5.5", + "version": "1.5.6", "description": "Background Removal in NodeJS", "resources": "@imgly/background-removal-node", "keywords": [ diff --git a/packages/web-data/.resources.mjs b/packages/web-data/.resources.mjs index bb35381..add4161 100644 --- a/packages/web-data/.resources.mjs +++ b/packages/web-data/.resources.mjs @@ -1,9 +1,28 @@ export default [ { path: '/onnxruntime-web/', - source: '../../node_modules/onnxruntime-web/dist/*.wasm', + source: + '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.wasm', mime: 'application/wasm' }, + { + path: '/onnxruntime-web/', + source: + '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.wasm', + mime: 'application/wasm' + }, + { + path: '/onnxruntime-web/', + source: + '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.mjs', + mime: 'text/javascript' + }, + { + path: '/onnxruntime-web/', + source: + '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.mjs', + mime: 'text/javascript' + }, { path: '/models/', source: '../../bundle/models/*', diff --git a/packages/web-data/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml b/packages/web-data/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml new file mode 100644 index 0000000..0d47201 --- /dev/null +++ b/packages/web-data/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml @@ -0,0 +1,9 @@ +--- +type: Changed +# type: Added +# type: Changed +# type: Removed +# type: Security +# private: true +description: | + Upgrade onnx-runtime diff --git a/packages/web-data/package.json b/packages/web-data/package.json index 410049d..23e15ba 100644 --- a/packages/web-data/package.json +++ b/packages/web-data/package.json @@ -1,6 +1,6 @@ { "name": "@imgly/background-removal-data", - "version": "1.5.5", + "version": "1.5.6", "description": "Background Removal Data", "keywords": [ "background-removal", @@ -46,6 +46,6 @@ "package:pack": "npm pack . --pack-destination ../../releases" }, "dependencies": { - "onnxruntime-web": "~1.18.0" + "onnxruntime-web": "~1.20.1" } } diff --git a/packages/web-examples/vite-project/.vercel/README.txt b/packages/web-examples/vite-project/.vercel/README.txt new file mode 100644 index 0000000..525d8ce --- /dev/null +++ b/packages/web-examples/vite-project/.vercel/README.txt @@ -0,0 +1,11 @@ +> Why do I have a folder named ".vercel" in my project? +The ".vercel" folder is created when you link a directory to a Vercel project. + +> What does the "project.json" file contain? +The "project.json" file contains: +- The ID of the Vercel project that you linked ("projectId") +- The ID of the user or team your Vercel project is owned by ("orgId") + +> Should I commit the ".vercel" folder? +No, you should not share the ".vercel" folder with anyone. +Upon creation, it will be automatically added to your ".gitignore" file. diff --git a/packages/web-examples/vite-project/.vercel/project.json b/packages/web-examples/vite-project/.vercel/project.json new file mode 100644 index 0000000..16b7408 --- /dev/null +++ b/packages/web-examples/vite-project/.vercel/project.json @@ -0,0 +1,4 @@ +{ + "orgId": "team_JFznad5UXNe4xKmEfdbZq5jl", + "projectId": "prj_jLTn0qO7rU5v6MEyaNREE9gIk3l2" +} diff --git a/packages/web-examples/vite-project/package-lock.json b/packages/web-examples/vite-project/package-lock.json index 5c871e5..a09db7c 100644 --- a/packages/web-examples/vite-project/package-lock.json +++ b/packages/web-examples/vite-project/package-lock.json @@ -1,12 +1,12 @@ { "name": "vite-project", - "version": "1.5.5", + "version": "1.5.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vite-project", - "version": "1.5.5", + "version": "1.5.6", "dependencies": { "@imgly/background-removal": "file:../../web", "@imgly/background-removal-data": "file:../../web-data", @@ -48,18 +48,18 @@ }, "../../web": { "name": "@imgly/background-removal", - "version": "1.5.5", + "version": "1.5.6", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { - "@types/lodash-es": "^4.17.12", - "@types/ndarray": "~1.0.14", - "@types/node": "~20.3.0", "lodash-es": "^4.17.21", "ndarray": "~1.0.0", - "onnxruntime-web": "~1.18.0", + "onnxruntime-web": "~1.20.1", "zod": "^3.23.8" }, "devDependencies": { + "@types/lodash-es": "^4.17.12", + "@types/ndarray": "~1.0.14", + "@types/node": "~20.3.0", "assert": "~2.0.0", "esbuild": "~0.18.0", "glob": "~10.3.0", @@ -75,10 +75,10 @@ }, "../../web-data": { "name": "@imgly/background-removal-data", - "version": "1.5.5", + "version": "1.5.6", "license": "SEE LICENSE IN LICENSE.md", "dependencies": { - "onnxruntime-web": "~1.18.0" + "onnxruntime-web": "~1.20.1" } }, "node_modules/@babel/parser": { diff --git a/packages/web-examples/vite-project/package.json b/packages/web-examples/vite-project/package.json index 99ddb2a..75b724a 100644 --- a/packages/web-examples/vite-project/package.json +++ b/packages/web-examples/vite-project/package.json @@ -1,7 +1,7 @@ { "name": "vite-project", "private": true, - "version": "1.5.5", + "version": "1.5.6", "type": "module", "scripts": { "start": "npm run dev", diff --git a/packages/web/CHANGELOG.md b/packages/web/CHANGELOG.md index a2fb480..e10154c 100644 --- a/packages/web/CHANGELOG.md +++ b/packages/web/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.6] + +### Changed + +- Upgrade onnx-runtime + ## [1.5.0] ### Added diff --git a/packages/web/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml b/packages/web/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml new file mode 100644 index 0000000..0d47201 --- /dev/null +++ b/packages/web/changelog/1.5.6/20242602190400-Added_ThirdPartyLicenses_json_Added.yaml @@ -0,0 +1,9 @@ +--- +type: Changed +# type: Added +# type: Changed +# type: Removed +# type: Security +# private: true +description: | + Upgrade onnx-runtime diff --git a/packages/web/package.json b/packages/web/package.json index 266819b..8a3fe98 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,6 +1,6 @@ { "name": "@imgly/background-removal", - "version": "1.5.5", + "version": "1.5.6", "description": "Background Removal in the Browser", "keywords": [ "background-removal", @@ -59,15 +59,15 @@ "lint": "npx prettier --write ." }, "dependencies": { - "@types/lodash-es": "^4.17.12", - "@types/ndarray": "~1.0.14", - "@types/node": "~20.3.0", "lodash-es": "^4.17.21", "ndarray": "~1.0.0", - "onnxruntime-web": "~1.18.0", + "onnxruntime-web": "~1.20.1", "zod": "^3.23.8" }, "devDependencies": { + "@types/lodash-es": "^4.17.12", + "@types/ndarray": "~1.0.14", + "@types/node": "~20.3.0", "assert": "~2.0.0", "esbuild": "~0.18.0", "glob": "~10.3.0", diff --git a/packages/web/src/onnx.ts b/packages/web/src/onnx.ts index 8a34b42..1b4f3f3 100644 --- a/packages/web/src/onnx.ts +++ b/packages/web/src/onnx.ts @@ -6,21 +6,18 @@ import type ORT from 'onnxruntime-web'; import * as ort_cpu from 'onnxruntime-web'; import * as ort_gpu from 'onnxruntime-web/webgpu'; -import { loadAsUrl } from './resource'; import * as caps from './capabilities'; import { Config } from './schema'; +import { loadAsUrl, resolveChunkUrls } from './resource'; async function createOnnxSession(model: any, config: Config) { const useWebGPU = config.device === 'gpu' && (await caps.webgpu()); - const useThreads = await caps.threads(); - const useSimd = caps.simd(); - const proxyToWorker = config.proxyToWorker; + // BUG: proxyToWorker is not working for WASM/CPU Backend for now + const proxyToWorker = useWebGPU && config.proxyToWorker; const executionProviders = [useWebGPU ? 'webgpu' : 'wasm']; const ort = useWebGPU ? ort_gpu : ort_cpu; if (config.debug) { - console.debug('\tUsing Threads:', useThreads); - console.debug('\tUsing SIMD:', useSimd); console.debug('\tUsing WebGPU:', useWebGPU); console.debug('\tProxy to Worker:', proxyToWorker); @@ -29,28 +26,20 @@ async function createOnnxSession(model: any, config: Config) { } ort.env.wasm.numThreads = caps.maxNumThreads(); - ort.env.wasm.simd = caps.simd(); ort.env.wasm.proxy = proxyToWorker; - const wasmPaths = { - 'ort-wasm-simd-threaded.wasm': useThreads && useSimd, - 'ort-wasm-simd.wasm': !useThreads && useSimd, - 'ort-wasm-threaded.wasm': !useWebGPU && useThreads && !useSimd, - 'ort-wasm.wasm': !useWebGPU && !useThreads && !useSimd - }; + // The path inside the resource bundle + const baseFilePath = useWebGPU + ? '/onnxruntime-web/ort-wasm-simd-threaded.jsep' + : '/onnxruntime-web/ort-wasm-simd-threaded'; - const proxiedWasmPaths = {}; - for (const [key, value] of Object.entries(wasmPaths)) { - if (value) { - const wasmPath = - useWebGPU && key.includes('simd') - ? `/onnxruntime-web/${key.replace('.wasm', '.jsep.wasm')}` - : `/onnxruntime-web/${key}`; - proxiedWasmPaths[key] = await loadAsUrl(wasmPath, config); - } - } + const wasmPath = await loadAsUrl(`${baseFilePath}.wasm`, config); + const mjsPath = await loadAsUrl(`${baseFilePath}.mjs`, config); - ort.env.wasm.wasmPaths = proxiedWasmPaths; + ort.env.wasm.wasmPaths = { + mjs: mjsPath, + wasm: wasmPath + }; if (config.debug) { console.debug('ort.env.wasm:', ort.env.wasm); diff --git a/packages/web/src/resource.ts b/packages/web/src/resource.ts index de90834..5f57ada 100644 --- a/packages/web/src/resource.ts +++ b/packages/web/src/resource.ts @@ -1,4 +1,4 @@ -export { loadAsBlob, loadAsUrl, preload }; +export { loadAsBlob, loadAsUrl, loadAsArrayBuffer, preload, resolveChunkUrls }; import { Config } from './schema'; @@ -24,6 +24,12 @@ async function preload(config: Config): Promise { async function loadAsUrl(url: string, config: Config): Promise { return URL.createObjectURL(await loadAsBlob(url, config)); } +async function loadAsArrayBuffer( + url: string, + config: Config +): Promise { + return await (await loadAsBlob(url, config)).arrayBuffer(); +} async function loadAsBlob(key: string, config: Config) { // load resource metadata @@ -49,8 +55,8 @@ async function loadAsBlob(key: string, config: Config) { const responses = chunks.map(async (chunk) => { const chunkSize = chunk.offsets[1] - chunk.offsets[0]; const url = config.publicPath - ? new URL(chunk.hash, config.publicPath).toString() - : chunk.hash; + ? new URL(chunk.name, config.publicPath).toString() + : chunk.name; const response = await fetch(url, config.fetchArgs); const blob = await response.blob(); @@ -79,3 +85,36 @@ async function loadAsBlob(key: string, config: Config) { } return data; } + +async function resolveChunkUrls(key: string, config: Config) { + // load resource metadata + const resourceUrl = new URL('resources.json', config.publicPath); + const resourceResponse = await fetch(resourceUrl); + if (!resourceResponse.ok) { + throw new Error( + `Resource metadata not found. Ensure that the config.publicPath is configured correctly.` + ); + } + const resourceMap = await resourceResponse.json(); + const entry = resourceMap[key]; + + if (!entry) { + throw new Error( + `Resource ${key} not found. Ensure that the config.publicPath is configured correctly.` + ); + } + + const chunks = entry.chunks; // list of entries + + const responses = chunks.map(async (chunk) => { + const url = config.publicPath + ? new URL(chunk.name, config.publicPath).toString() + : chunk.name; + + return url; + }); + // we could create a new buffer here and use the chunk entries and combine the file instead + + const allUrls = await Promise.all(responses); + return allUrls; +} diff --git a/scripts/package-resources.mjs b/scripts/package-resources.mjs index 31d3885..8fef023 100644 --- a/scripts/package-resources.mjs +++ b/scripts/package-resources.mjs @@ -117,25 +117,28 @@ function isFunction(variable) { return typeof variable === 'function'; } -const ChunkSize = 4 * 1024 * 1024; // 4 MB chunks async function transform(fileName, entry) { - const buffer = Buffer.alloc(ChunkSize); + const DefaultChunkSize = 4 * 1024 * 1024; // 4 MB chunks + + const chunkSize = entry.chunkSize || DefaultChunkSize; + const buffer = Buffer.alloc(chunkSize); const fileSize = await sizeFile(fileName); const fileHandle = fs.openSync(fileName); const chunks = []; - for (let offset = 0; offset < fileSize; offset += ChunkSize) { - const bytesRead = fs.readSync(fileHandle, buffer, 0, ChunkSize, offset); + for (let offset = 0; offset < fileSize; offset += chunkSize) { + const bytesRead = fs.readSync(fileHandle, buffer, 0, chunkSize, offset); const data = buffer.subarray(0, bytesRead); const hash = crypto.createHash('sha256'); hash.update(data); const chunkHash = hash.digest('hex'); - const url = chunkHash; - const destFile = path.resolve(DefaultOutDir, chunkHash); + const name = entry.name ? entry.name : chunkHash; + const destFile = path.resolve(DefaultOutDir, name); fs.writeFileSync(destFile, data); chunks.push({ hash: chunkHash, + name: name, offsets: [offset, offset + bytesRead] }); }