diff --git a/README.md b/README.md index 16c4fa1..6462c3d 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,16 @@ await collector.onComplete(); `addDataPoint` throws if a background upload failed. Values are rounded to two decimals. +To store the unit of a time-series, pass an object instead of the name (see [Units](#units)): + +```js +const collector = await edgeML.datasetCollector(url, key, "walk-1", true, [ + { name: "accX", unit: "g", quantity: "acceleration" }, + { name: "temp", unit: "degF", quantity: "temperature" }, + "battery", // plain names still work: unit unknown +]); +``` + ### Upload a whole dataset ```js @@ -91,6 +101,7 @@ const datasetId = await edgeML.sendDataset("backendUrl", "deviceApiKey", { name: "datasetName", timeSeries: [ { name: "accX", data: [[1618760114000, 1.23], [1618760114020, 1.25]] }, // [unix ms, value] + { name: "accY", unit: "g", quantity: "acceleration", data: [[1618760114000, 0.98]] }, // with unit ], metaData: { key: "value" }, // optional labeling: "labeling_label", // optional @@ -163,6 +174,27 @@ await predictor.predictWindow(rows); // classify your own window: win predictor.reset(); // clear the buffer ``` +### Sensor units + +Every model knows the unit it expects for each input (`predictor.inputUnits`). Tell the predictor which unit your sensors deliver, and it converts the values before predicting: + +```js +const predictor = await edgeML.loadModel(url, key, modelId, { + units: { accX: "g", accY: "g", accZ: "g", temp: "degF" }, +}); +predictor.addSample({ accX: 0.02, accY: 1.0, accZ: 0.01, temp: 71.6 }); // in g and °F + +predictor.inputUnits; +// [{ name: "accX", quantity: "acceleration", unit: "m/s^2", conversions: { "g": { scale: 9.80665, offset: 0 }, ... } }, ...] + +predictor.setUnits({ accX: "m/s^2" }); // change the declaration later +``` + +- Sensors you don't declare are passed through unchanged, as are all inputs of models trained before units existed (or on data of unknown unit). +- If the model can't convert from a declared unit (e.g. `"degC"` for an acceleration), `loadModel`/`setUnits` throw a `UnitError` that lists the units it accepts, instead of predicting on wrong values. +- Units are matched by catalog id (`"m/s^2"`, `"degF"`, ...). Pass the catalog to also accept labels and aliases like `"m/s²"` or `"°F"`: `{ units, catalog: await edgeML.fetchUnits(url) }`. +- `predictWindow` takes values in the declared units too. + ### Choosing the ONNX runtime - **Node.js**: `onnxruntime-node` is loaded automatically, with `onnxruntime-web` as a fallback. @@ -178,6 +210,22 @@ OnnxPredictor.setRuntime(ort); // or per model: loadModel(url, key, id, { ort }) `sessionOptions` (e.g. `{ executionProviders: ["webgpu", "wasm"] }`) can be passed the same way and are forwarded to `ort.InferenceSession.create`. +## Units + +edge-ml stores a quantity and a unit for each time-series. `quantity` is an id of the server's unit catalog (e.g. `acceleration`, `angular_velocity`, `temperature`) or a name of your own, and `unit` is one of its units (e.g. `g`, `rad/s`, `degF`). When a model is trained, series of catalog quantities are converted to the quantity's default unit (e.g. m/s² for acceleration), so datasets recorded in different units can be trained on together. Series without a quantity are trained on as they are. + +```js +const catalog = await edgeML.fetchUnits("https://app.edge-ml.org"); // no key needed +catalog.quantities.map((q) => [q.id, q.defaultUnit, q.units.map((u) => u.id)]); +// [["acceleration", "m/s^2", ["m/s^2", "g", "mg", "cm/s^2"]], ["angular_velocity", "deg/s", ["deg/s", "rad/s", "rpm"]], ...] +``` + +## Unreleased: units + +- Time-series can carry a unit and quantity when uploading (`{ name, unit, quantity }` instead of a name). +- `OnnxPredictor` converts declared sensor units to the units the model expects (`units` option, `setUnits`, `inputUnits`), and throws a `UnitError` for units it can't convert. +- `fetchUnits(url)` returns the server's unit catalog. + ## Changes from 4.x - Node.js >= 18 is required (the library uses the built-in `fetch`; axios is no longer a dependency). diff --git a/__tests__/fixtures/README.md b/__tests__/fixtures/README.md index 0f36981..e926ee8 100644 --- a/__tests__/fixtures/README.md +++ b/__tests__/fixtures/README.md @@ -14,3 +14,9 @@ The JS tests check that `OnnxPredictor` reproduces both. To regenerate them, run the backend's ONNX exporter on the same pipelines and write `model.onnx`, `manifest.json` and `reference.json` per model. + +The manifests above predate units, so they also cover old models. For the unit +tests, `input_units.json` is the `input.units` block the backend +(`ml/app/units/deploy.py`) writes for a model trained on x in m/s^2 +(acceleration), y in degC (temperature) and z in a custom unit, and +`unit_catalog.json` is the backend's `GET /ml/units/` response. diff --git a/__tests__/fixtures/input_units.json b/__tests__/fixtures/input_units.json new file mode 100644 index 0000000..1da093c --- /dev/null +++ b/__tests__/fixtures/input_units.json @@ -0,0 +1,50 @@ +[ + { + "name": "x", + "quantity": "acceleration", + "unit": "m/s^2", + "conversions": { + "m/s^2": { + "scale": 1.0, + "offset": 0.0 + }, + "g": { + "scale": 9.80665, + "offset": 0.0 + }, + "mg": { + "scale": 0.00980665, + "offset": 0.0 + }, + "cm/s^2": { + "scale": 0.01, + "offset": 0.0 + } + } + }, + { + "name": "y", + "quantity": "temperature", + "unit": "degC", + "conversions": { + "degC": { + "scale": 1.0, + "offset": 0.0 + }, + "K": { + "scale": 1.0, + "offset": -273.15 + }, + "degF": { + "scale": 0.5555555555555556, + "offset": -17.77777777777778 + } + } + }, + { + "name": "z", + "quantity": "strain", + "unit": "ue", + "conversions": {} + } +] diff --git a/__tests__/fixtures/unit_catalog.json b/__tests__/fixtures/unit_catalog.json new file mode 100644 index 0000000..957b080 --- /dev/null +++ b/__tests__/fixtures/unit_catalog.json @@ -0,0 +1,614 @@ +{ + "version": 1, + "quantities": [ + { + "id": "acceleration", + "label": "Acceleration", + "defaultUnit": "m/s^2", + "units": [ + { + "id": "m/s^2", + "label": "m/s²", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "m/s²", + "m/s2" + ] + }, + { + "id": "g", + "label": "g", + "scale": 9.80665, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mg", + "label": "mg", + "scale": 0.00980665, + "offset": 0.0, + "aliases": [] + }, + { + "id": "cm/s^2", + "label": "cm/s²", + "scale": 0.01, + "offset": 0.0, + "aliases": [ + "cm/s²", + "Gal" + ] + } + ] + }, + { + "id": "angular_velocity", + "label": "Angular velocity", + "defaultUnit": "deg/s", + "units": [ + { + "id": "deg/s", + "label": "°/s", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "°/s", + "dps" + ] + }, + { + "id": "rad/s", + "label": "rad/s", + "scale": 57.29577951308232, + "offset": 0.0, + "aliases": [] + }, + { + "id": "rpm", + "label": "rpm", + "scale": 6.0, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "angle", + "label": "Angle", + "defaultUnit": "deg", + "units": [ + { + "id": "deg", + "label": "°", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "°" + ] + }, + { + "id": "rad", + "label": "rad", + "scale": 57.29577951308232, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "temperature", + "label": "Temperature", + "defaultUnit": "degC", + "units": [ + { + "id": "degC", + "label": "°C", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "°C", + "C" + ] + }, + { + "id": "K", + "label": "K", + "scale": 1.0, + "offset": -273.15, + "aliases": [] + }, + { + "id": "degF", + "label": "°F", + "scale": 0.5555555555555556, + "offset": -17.77777777777778, + "aliases": [ + "°F", + "F" + ] + } + ] + }, + { + "id": "pressure", + "label": "Pressure", + "defaultUnit": "hPa", + "units": [ + { + "id": "Pa", + "label": "Pa", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "hPa", + "label": "hPa", + "scale": 100.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "kPa", + "label": "kPa", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mbar", + "label": "mbar", + "scale": 100.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "bar", + "label": "bar", + "scale": 100000.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "atm", + "label": "atm", + "scale": 101325.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "psi", + "label": "psi", + "scale": 6894.757293168, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mmHg", + "label": "mmHg", + "scale": 133.322387415, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "magnetic_flux_density", + "label": "Magnetic flux density", + "defaultUnit": "uT", + "units": [ + { + "id": "T", + "label": "T", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mT", + "label": "mT", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "uT", + "label": "µT", + "scale": 1e-06, + "offset": 0.0, + "aliases": [ + "µT", + "μT" + ] + }, + { + "id": "nT", + "label": "nT", + "scale": 1e-09, + "offset": 0.0, + "aliases": [] + }, + { + "id": "G", + "label": "G (gauss)", + "scale": 0.0001, + "offset": 0.0, + "aliases": [ + "gauss" + ] + }, + { + "id": "mG", + "label": "mG", + "scale": 1e-07, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "relative_humidity", + "label": "Relative humidity", + "defaultUnit": "%", + "units": [ + { + "id": "%", + "label": "%", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "%RH" + ] + }, + { + "id": "fraction", + "label": "fraction (0–1)", + "scale": 100.0, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "voltage", + "label": "Voltage", + "defaultUnit": "V", + "units": [ + { + "id": "V", + "label": "V", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mV", + "label": "mV", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "uV", + "label": "µV", + "scale": 1e-06, + "offset": 0.0, + "aliases": [ + "µV", + "μV" + ] + }, + { + "id": "kV", + "label": "kV", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "current", + "label": "Electric current", + "defaultUnit": "mA", + "units": [ + { + "id": "A", + "label": "A", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mA", + "label": "mA", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "uA", + "label": "µA", + "scale": 1e-06, + "offset": 0.0, + "aliases": [ + "µA", + "μA" + ] + } + ] + }, + { + "id": "length", + "label": "Length / distance", + "defaultUnit": "m", + "units": [ + { + "id": "m", + "label": "m", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "cm", + "label": "cm", + "scale": 0.01, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mm", + "label": "mm", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "km", + "label": "km", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "in", + "label": "in", + "scale": 0.0254, + "offset": 0.0, + "aliases": [] + }, + { + "id": "ft", + "label": "ft", + "scale": 0.3048, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "velocity", + "label": "Velocity", + "defaultUnit": "m/s", + "units": [ + { + "id": "m/s", + "label": "m/s", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "km/h", + "label": "km/h", + "scale": 0.2777777777777778, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mph", + "label": "mph", + "scale": 0.44704, + "offset": 0.0, + "aliases": [] + }, + { + "id": "kn", + "label": "kn", + "scale": 0.5144444444444445, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "frequency", + "label": "Frequency / rate", + "defaultUnit": "Hz", + "units": [ + { + "id": "Hz", + "label": "Hz", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "kHz", + "label": "kHz", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "bpm", + "label": "bpm (per minute)", + "scale": 0.016666666666666666, + "offset": 0.0, + "aliases": [ + "1/min" + ] + } + ] + }, + { + "id": "illuminance", + "label": "Illuminance", + "defaultUnit": "lx", + "units": [ + { + "id": "lx", + "label": "lx", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "lux" + ] + } + ] + }, + { + "id": "sound_level", + "label": "Sound level", + "defaultUnit": "dB", + "units": [ + { + "id": "dB", + "label": "dB", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "dBA", + "dB(A)" + ] + } + ] + }, + { + "id": "mass", + "label": "Mass", + "defaultUnit": "kg", + "units": [ + { + "id": "kg", + "label": "kg", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "g", + "label": "g", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "force", + "label": "Force", + "defaultUnit": "N", + "units": [ + { + "id": "N", + "label": "N", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "kN", + "label": "kN", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "power", + "label": "Power", + "defaultUnit": "W", + "units": [ + { + "id": "W", + "label": "W", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "mW", + "label": "mW", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "kW", + "label": "kW", + "scale": 1000.0, + "offset": 0.0, + "aliases": [] + } + ] + }, + { + "id": "time", + "label": "Duration", + "defaultUnit": "s", + "units": [ + { + "id": "s", + "label": "s", + "scale": 1.0, + "offset": 0.0, + "aliases": [] + }, + { + "id": "ms", + "label": "ms", + "scale": 0.001, + "offset": 0.0, + "aliases": [] + }, + { + "id": "us", + "label": "µs", + "scale": 1e-06, + "offset": 0.0, + "aliases": [ + "µs", + "μs" + ] + } + ] + }, + { + "id": "ratio", + "label": "Ratio / dimensionless", + "defaultUnit": "1", + "units": [ + { + "id": "1", + "label": "1 (unitless)", + "scale": 1.0, + "offset": 0.0, + "aliases": [ + "" + ] + }, + { + "id": "%", + "label": "%", + "scale": 0.01, + "offset": 0.0, + "aliases": [] + } + ] + } + ] +} diff --git a/__tests__/test_exports.js b/__tests__/test_exports.js index 1607619..b0e49b7 100644 --- a/__tests__/test_exports.js +++ b/__tests__/test_exports.js @@ -7,7 +7,7 @@ const edgeML = require("../dist/index"); describe("Package exports", () => { it("exposes the documented API", () => { for (const name of [ - "datasetCollector", "sendDataset", + "datasetCollector", "sendDataset", "fetchUnits", "UnitError", "listPipelines", "trainModel", "startTraining", "waitForModel", "getModel", "listModels", "loadModel", "OnnxPredictor", "PredictorError", ]) { diff --git a/__tests__/test_units.js b/__tests__/test_units.js new file mode 100644 index 0000000..2165a4f --- /dev/null +++ b/__tests__/test_units.js @@ -0,0 +1,204 @@ +const fs = require("fs"); +const path = require("path"); +const edgeML = require("../dist/index"); +const { OnnxPredictor, UnitError } = edgeML; + +const FIXTURES = path.join(__dirname, "fixtures"); +const MODEL = path.join(FIXTURES, "random_forest", "model.onnx"); +const OLD_MANIFEST = JSON.parse(fs.readFileSync(path.join(FIXTURES, "random_forest", "manifest.json"))); +const INPUT_UNITS = require(path.join(FIXTURES, "input_units.json")); +const CATALOG = require(path.join(FIXTURES, "unit_catalog.json")); +const ref = require(path.join(FIXTURES, "random_forest", "reference.json")); + +// The same model, as exported by a backend with units: +// x in m/s^2 (acceleration), y in degC (temperature), z in a custom unit "ue". +const MANIFEST = { ...OLD_MANIFEST, input: { ...OLD_MANIFEST.input, units: INPUT_UNITS } }; +const G = 9.80665; + +const load = (manifest, opts) => OnnxPredictor.load(fs.readFileSync(MODEL), manifest, opts); + +// Records what the model is fed by wrapping session.run. +function recordInputs(predictor) { + const inputs = []; + const run = predictor.session.run.bind(predictor.session); + predictor.session.run = (feeds) => { + const t = feeds[predictor.manifest.input.name]; + const n = predictor.sensors.length; + inputs.push(Array.from({ length: t.dims[1] }, (_, r) => Array.from(t.data.slice(r * n, (r + 1) * n)))); + return run(feeds); + }; + return inputs; +} + +describe("OnnxPredictor units", () => { + it("reads the expected units from the manifest", async () => { + const p = await load(MANIFEST); + expect(p.inputUnits.map((u) => [u.name, u.quantity, u.unit])).toEqual([ + ["x", "acceleration", "m/s^2"], + ["y", "temperature", "degC"], + ["z", "strain", "ue"], + ]); + expect(p.inputUnits[0].conversions.g).toEqual({ scale: G, offset: 0 }); + }); + + it("converts declared units to the units the model expects", async () => { + const p = await load(MANIFEST, { units: { x: "g", y: "degF", z: "ue" } }); + const inputs = recordInputs(p); + const window = ref.windows[0]; + // the same physical values, delivered in g and degF + const delivered = window.map(([x, y, z]) => [x / G, (y * 9) / 5 + 32, z]); + const converted = await p.predictWindow(delivered); + inputs[0].forEach((row, r) => { + expect(row[0]).toBeCloseTo(window[r][0], 4); + expect(row[1]).toBeCloseTo(window[r][1], 4); + expect(row[2]).toBeCloseTo(window[r][2], 4); + }); + + const plain = await load(MANIFEST); + expect(converted.prediction).toBe((await plain.predictWindow(window)).prediction); + }); + + it("converts buffered datapoints too", async () => { + const p = await load(MANIFEST, { units: { x: "mg" } }); + const inputs = recordInputs(p); + ref.timestamps[0].forEach((t, r) => { + const [x, y, z] = ref.windows[0][r]; + p.addSample({ x: (x / G) * 1000, y, z }, t); + }); + await p.predict(); + inputs[0].forEach((row, r) => expect(row[0]).toBeCloseTo(ref.windows[0][r][0], 4)); + }); + + it("passes undeclared sensors and the expected unit through unchanged", async () => { + const p = await load(MANIFEST, { units: { x: "m/s^2" } }); + const inputs = recordInputs(p); + await p.predictWindow(ref.windows[0]); + inputs[0].forEach((row, r) => row.forEach((v, c) => expect(v).toBeCloseTo(ref.windows[0][r][c], 5))); + }); + + it("throws a UnitError for units the model cannot convert from", async () => { + await expect(load(MANIFEST, { units: { x: "degC" } })).rejects.toBeInstanceOf(UnitError); + await expect(load(MANIFEST, { units: { x: "furlong" } })).rejects.toThrow( + "'x' is declared in 'furlong', but the model expects m/s^2 (acceleration) and cannot convert from 'furlong'. Accepted units: m/s^2, g, mg, cm/s^2." + ); + // custom quantity: only its own unit is accepted + await expect(load(MANIFEST, { units: { z: "mm/m" } })).rejects.toThrow("Deliver it in ue"); + + const p = await load(MANIFEST); + expect(() => p.setUnits({ y: "rad" })).toThrow(UnitError); + expect(() => p.setUnits({ y: 5 })).toThrow(TypeError); + }); + + it("accepts unit labels and aliases with the catalog", async () => { + await expect(load(MANIFEST, { units: { y: "°F" } })).rejects.toBeInstanceOf(UnitError); + const p = await load(MANIFEST, { units: { x: "m/s²", y: "°F" }, catalog: CATALOG }); + const inputs = recordInputs(p); + await p.predictWindow(ref.windows[0].map(([x, y, z]) => [x, (y * 9) / 5 + 32, z])); + inputs[0].forEach((row, r) => expect(row[1]).toBeCloseTo(ref.windows[0][r][1], 4)); + }); + + it("ignores units of sensors the model does not use", async () => { + await expect(load(MANIFEST, { units: { gyroX: "rad/s" } })).resolves.toBeInstanceOf(OnnxPredictor); + }); + + it("setUnits replaces earlier declarations", async () => { + const p = await load(MANIFEST, { units: { x: "g" } }); + p.setUnits({}); + const inputs = recordInputs(p); + await p.predictWindow(ref.windows[0]); + expect(inputs[0][0][0]).toBeCloseTo(ref.windows[0][0][0], 5); + }); +}); + +describe("Models without units (backwards compatibility)", () => { + it("reports every input as unknown", async () => { + const p = await load(OLD_MANIFEST); + expect(p.inputUnits).toEqual([ + { name: "x", quantity: null, unit: "", conversions: {} }, + { name: "y", quantity: null, unit: "", conversions: {} }, + { name: "z", quantity: null, unit: "", conversions: {} }, + ]); + }); + + it("passes values through even when the app declares units", async () => { + const p = await load(OLD_MANIFEST, { units: { x: "g", y: "degF" } }); + const res = await p.predictWindow(ref.windows[0]); + res.scores.forEach((s, j) => expect(s).toBeCloseTo(ref.onnx_scores[0][j], 4)); + }); + + it("treats inputs with unknown units in a new manifest the same way", async () => { + const units = INPUT_UNITS.map((u) => ({ ...u, quantity: null, unit: "", conversions: {} })); + const p = await load({ ...OLD_MANIFEST, input: { ...OLD_MANIFEST.input, units } }, { units: { x: "g" } }); + const res = await p.predictWindow(ref.windows[0]); + res.scores.forEach((s, j) => expect(s).toBeCloseTo(ref.onnx_scores[0][j], 4)); + }); +}); + +describe("Units on upload and the unit catalog", () => { + function mockFetch(routes) { + const calls = []; + global.fetch = jest.fn(async (url, init = {}) => { + const body = init.body ? JSON.parse(init.body) : undefined; + calls.push({ url, method: init.method || "GET", body }); + const handler = routes[`${init.method || "GET"} ${url.replace("https://edge-ml.example", "")}`]; + if (!handler) return new Response(JSON.stringify({ detail: "no route" }), { status: 404 }); + return new Response(JSON.stringify(handler(body)), { status: 200 }); + }); + return calls; + } + afterEach(() => delete global.fetch); + + const URL = "https://edge-ml.example"; + const routes = { + "POST /api/v1/deviceapi/dataset/init/W": () => ({ id: "ds1" }), + "POST /api/v1/deviceapi/dataset/append/W/ds1": () => null, + }; + + it("datasetCollector sends units and keeps plain names plain", async () => { + const calls = mockFetch(routes); + const c = await edgeML.datasetCollector(URL, "W", "rec", false, [ + "plain", + { name: "accX", unit: "g", quantity: "acceleration" }, + { name: "strain", unit: "ue" }, + ]); + expect(calls[0].body.timeSeries).toEqual([ + "plain", + { name: "accX", unit: "g", quantity: "acceleration" }, + { name: "strain", unit: "ue", quantity: null }, + ]); + c.addDataPoint(1, "accX", 1); + c.addDataPoint(1, "plain", 2); + expect(() => c.addDataPoint(1, "accY", 1)).toThrow("invalid time-series name"); + await c.onComplete(); + expect(calls[1].body.data.map((d) => d.name)).toEqual(["accX", "plain"]); + }); + + it("datasetCollector rejects series without a name", async () => { + mockFetch(routes); + await expect(edgeML.datasetCollector(URL, "W", "rec", true, [{ unit: "g" }])).rejects.toThrow("needs a name"); + await expect(edgeML.datasetCollector(URL, "W", "rec", true, [])).rejects.toThrow("non-empty"); + }); + + it("sendDataset sends units on init and only data on append", async () => { + const calls = mockFetch(routes); + await edgeML.sendDataset(URL, "W", { + name: "whole", + timeSeries: [ + { name: "temp", unit: "degF", quantity: "temperature", data: [[1, 70]] }, + { name: "accX", data: [[1, 0.5]] }, + ], + }); + expect(calls[0].body.timeSeries).toEqual([{ name: "temp", unit: "degF", quantity: "temperature" }, "accX"]); + expect(calls[1].body.data).toEqual([ + { name: "temp", data: [[1, 70]] }, + { name: "accX", data: [[1, 0.5]] }, + ]); + }); + + it("fetchUnits returns the catalog", async () => { + const calls = mockFetch({ "GET /ml/units/": () => CATALOG }); + const catalog = await edgeML.fetchUnits(URL + "/"); + expect(catalog.quantities.find((q) => q.id === "acceleration").defaultUnit).toBe("m/s^2"); + expect(calls[0].url).toBe("https://edge-ml.example/ml/units/"); + }); +}); diff --git a/dist/index.browser.esm.js b/dist/index.browser.esm.js index 04cb830..3bac5f3 100644 --- a/dist/index.browser.esm.js +++ b/dist/index.browser.esm.js @@ -289,6 +289,105 @@ function fetchOnnx(url, key, modelId) { ]); } +/** + * Thrown when a sensor unit cannot be converted to the unit a model expects, + * so the app gets an error instead of wrong predictions. + */ +class UnitError extends Error {} + +/** + * Fetches the unit catalog of the edge-ml server: every quantity with its + * units and the unit models expect (`defaultUnit`). No API key needed. + * @param {string} url - The url of the backend server + * @returns {Promise} + */ +function fetchUnits(url) { + return http.get(url, "/ml/units/"); +} + +/** + * Time-series as given by the app: a plain name, or `{ name, unit?, quantity? }`. + * @param {string | { name: string, unit?: string, quantity?: string }} ts + * @returns {{ name: string, unit?: string, quantity?: string }} + */ +function seriesSpec(ts) { + if (typeof ts === "string") return { name: ts }; + if (!ts || typeof ts.name !== "string" || !ts.name) { + throw new TypeError("A time-series needs a name, e.g. \"accX\" or { name: \"accX\", unit: \"g\", quantity: \"acceleration\" }"); + } + return ts; +} + +/** + * The time-series entry of a dataset init request. Plain names stay plain + * strings, so servers without unit support keep accepting them. + */ +function initEntry(ts) { + const { name, unit, quantity } = seriesSpec(ts); + if (!unit && !quantity) return name; + return { name, unit: unit || "", quantity: quantity || null }; +} + +/** + * The expected unit of every model input, from `manifest.input.units`. + * Exports made before units existed have none: every input is unknown. + * @returns {import("../index").InputUnit[]} + */ +function manifestUnits(manifest) { + const names = manifest.input.timeseries; + const byName = {}; + for (const u of manifest.input.units || []) byName[u.name] = u; + return names.map((name) => { + const u = byName[name] || {}; + return { name, quantity: u.quantity || null, unit: u.unit || "", conversions: u.conversions || {} }; + }); +} + +const isUnknown = (input) => !input.quantity && !input.unit; + +/** The catalog unit id of `unit` within `quantityId`, matching ids, labels and aliases. */ +function catalogUnitId(catalog, quantityId, unit) { + const quantity = catalog && (catalog.quantities || []).find((q) => q.id === quantityId); + if (!quantity) return null; + const match = quantity.units.find((u) => u.id === unit || u.label === unit || (u.aliases || []).includes(unit)); + return match ? match.id : null; +} + +/** + * Works out how to convert each input from the unit the app delivers to the + * unit the model expects. + * + * @param {import("../index").InputUnit[]} inputs - from `manifestUnits` + * @param {{ [sensor: string]: string }} sensorUnits - unit each sensor delivers; sensors the model does not use are ignored + * @param {import("../index").UnitCatalog} [catalog] - optional, to also accept unit labels and aliases (e.g. "°C") + * @returns {({ scale: number, offset: number } | null)[]} per input, null = pass values through + */ +function resolveConversions(inputs, sensorUnits, catalog) { + return inputs.map((input) => { + const unit = sensorUnits ? sensorUnits[input.name] : undefined; + if (unit === undefined || unit === null) return null; + if (typeof unit !== "string") throw new TypeError(`The unit of '${input.name}' must be a string`); + // trained before units existed, or on data of unknown unit: values go in as they are + if (isUnknown(input)) return null; + + let conv = input.conversions[unit]; + if (!conv && unit !== input.unit && input.quantity) { + const id = catalogUnitId(catalog, input.quantity, unit); + if (id) conv = input.conversions[id] || (id === input.unit ? { scale: 1, offset: 0 } : undefined); + } + if (!conv && unit === input.unit) conv = { scale: 1, offset: 0 }; + if (!conv) { + const accepted = Object.keys(input.conversions); + const expected = `${input.unit || "no unit"}${input.quantity ? ` (${input.quantity})` : ""}`; + throw new UnitError( + `'${input.name}' is declared in '${unit}', but the model expects ${expected} and cannot convert from '${unit}'. ` + + (accepted.length > 1 ? `Accepted units: ${accepted.join(", ")}.` : `Deliver it in ${input.unit || "the unit it was trained on"}.`) + ); + } + return conv.scale === 1 && conv.offset === 0 ? null : { scale: conv.scale, offset: conv.offset }; + }); +} + let runtime = null; async function getRuntime(ort) { @@ -318,8 +417,9 @@ class OnnxPredictor { * @param {any} session - ort.InferenceSession * @param {any} manifest - manifest.json of the export * @param {any} ort - the onnxruntime module + * @param {{ units?: { [sensor: string]: string }, catalog?: object }} [unitOptions] */ - constructor(session, manifest, ort) { + constructor(session, manifest, ort, { units, catalog } = {}) { if (!manifest || manifest.format !== "onnx") throw new Error("Not an edge-ml ONNX manifest"); this.session = session; this.manifest = manifest; @@ -331,6 +431,28 @@ class OnnxPredictor { /** @type {number} */ this.windowSize = manifest.window.size; this.buffer = new SampleBuffer(this.sensors, this.windowSize); + /** + * Unit the model expects for each input, with conversions from other units. + * Inputs of models trained before units existed have quantity null and unit "". + * @type {import("../../index").InputUnit[]} + */ + this.inputUnits = manifestUnits(manifest); + this._conversions = this.sensors.map(() => null); + this._catalog = catalog; + if (units) this.setUnits(units); + } + + /** + * Declares the unit each sensor delivers, e.g. `{ accX: "g" }`. Values are + * then converted to the unit the model expects before predicting. Sensors + * left out (and inputs of models without unit information) are passed through. + * Throws a UnitError if the model cannot convert from a declared unit. + * @param {{ [sensor: string]: string }} units + * @param {object} [catalog] - from `fetchUnits`, to also accept unit labels and aliases like "°C" + */ + setUnits(units, catalog = this._catalog) { + this._conversions = resolveConversions(this.inputUnits, units || {}, catalog); + this._catalog = catalog; } /** @@ -344,19 +466,21 @@ class OnnxPredictor { /** * @param {ArrayBuffer | Uint8Array} model - bytes of model.onnx * @param {object} manifest - parsed manifest.json - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {{ ort?: any, sessionOptions?: object, units?: { [sensor: string]: string }, catalog?: object }} [opts] */ - static async load(model, manifest, { ort, sessionOptions } = {}) { + static async load(model, manifest, { ort, sessionOptions, units, catalog } = {}) { + // check the units before creating the session, so a mistake fails fast + if (units) resolveConversions(manifestUnits(manifest), units, catalog); const rt = await getRuntime(ort); const session = await rt.InferenceSession.create(toBytes(model), sessionOptions); - return new OnnxPredictor(session, manifest, rt); + return new OnnxPredictor(session, manifest, rt, { units, catalog }); } /** * Loads model.onnx and manifest.json from URLs (e.g. an unzipped download from edge-ml). * @param {string} modelUrl * @param {string} manifestUrl - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromUrls(modelUrl, manifestUrl, opts) { const [model, manifest] = await Promise.all([ @@ -370,7 +494,7 @@ class OnnxPredictor { * Node.js only: loads model.onnx and manifest.json from disk. * @param {string} modelPath * @param {string} manifestPath - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromFiles(modelPath, manifestPath, opts) { const [model, manifest] = await Promise.all([readFile(), readFile()]); @@ -382,7 +506,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromServer(url, key, modelId, opts) { const [model, manifest] = await fetchOnnx(url, key, modelId); @@ -423,7 +547,7 @@ class OnnxPredictor { /** * Classifies one window. - * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order + * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order, in the declared units * @returns {Promise} */ async predictWindow(window) { @@ -431,7 +555,13 @@ class OnnxPredictor { if (window.length !== this.windowSize || window.some((row) => row.length !== n)) { throw new TypeError(`Window must have shape [${this.windowSize}][${n}]`); } - const data = Float32Array.from(window.flat()); + const data = new Float32Array(this.windowSize * n); + window.forEach((row, r) => + row.forEach((value, c) => { + const conv = this._conversions[c]; + data[r * n + c] = conv ? value * conv.scale + conv.offset : value; + }) + ); const input = new this.ort.Tensor("float32", data, [1, this.windowSize, n]); const outputs = await this.session.run({ [this.manifest.input.name]: input }); const scores = Array.from(outputs[this.manifest.output.name].data); @@ -453,7 +583,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); @@ -470,8 +600,10 @@ const URLS = { * Uploads a whole dataset to a specific project * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key (write access) - * @param {{ name: string, timeSeries: { name: string, data: [number, number][] }[], metaData?: { [key: string]: string }, labeling?: string }} dataset - * time-series data as [unix ms, value] pairs; labeling as "{labeling}_{label}" for the whole dataset + * @param {import("../index").DatasetUpload} dataset + * time-series data as [unix ms, value] pairs, each with an optional unit and quantity + * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as + * "{labeling}_{label}" for the whole dataset * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { @@ -481,7 +613,7 @@ async function sendDataset(url, key, dataset) { const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map((ts) => ts.name), + timeSeries: dataset.timeSeries.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -502,7 +634,12 @@ async function sendDataset(url, key, dataset) { * * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key + * @param {string} name - Name of the new dataset * @param {boolean} useDeviceTime - True if you want to use timestamps generated by the server + * @param {(string | { name: string, unit?: string, quantity?: string })[]} timeSeries - names, or + * objects with the unit and quantity of the series, e.g. { name: "accX", unit: "g", quantity: "acceleration" } + * @param {{ [key: string]: string }} [metaData] + * @param {string} [datasetLabel] - label for the whole dataset as "{labeling}_{label}" * @returns Function to upload single datapoints to one dataset inside a specific project */ async function datasetCollector( @@ -519,10 +656,15 @@ async function datasetCollector( labeling = {"labelingName": datasetLabel.split("_")[0], "labelName": datasetLabel.split("_")[1]}; } + if (!Array.isArray(timeSeries) || timeSeries.length === 0) { + throw new Error("timeSeries must be a non-empty array"); + } + const seriesNames = timeSeries.map((ts) => seriesSpec(ts).name); + const data = await http.post(url, URLS.initDatasetIncrement + key, { name: name, - metaData: metaData, - timeSeries: timeSeries, + metaData: metaData || {}, + timeSeries: timeSeries.map(initEntry), labeling: labeling }); if (!data || !data.id) { @@ -534,7 +676,6 @@ async function datasetCollector( var dataStore = { data: [] }; var lastChecked = Date.now(); var error = undefined; - var timeSeries = timeSeries; /** * Uploads a vlaue for a specific timestamp to a datasets timeSeries with name sensorName @@ -545,7 +686,7 @@ async function datasetCollector( */ function addDataPoint(time, name, value) { - if (!timeSeries.includes(name)) { + if (!seriesNames.includes(name)) { throw Error("invalid time-series name") } @@ -632,6 +773,9 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + // units + fetchUnits: fetchUnits, + UnitError: UnitError, // training on the edge-ml server listPipelines: listPipelines, trainModel: trainModel, @@ -645,4 +789,4 @@ const edgeML = { PredictorError: PredictorError, }; -export { OnnxPredictor, PredictorError, datasetCollector, edgeML as default, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; +export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, fetchUnits, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; diff --git a/dist/index.browser.js b/dist/index.browser.js index ad79e27..841f613 100644 --- a/dist/index.browser.js +++ b/dist/index.browser.js @@ -295,6 +295,105 @@ ]); } + /** + * Thrown when a sensor unit cannot be converted to the unit a model expects, + * so the app gets an error instead of wrong predictions. + */ + class UnitError extends Error {} + + /** + * Fetches the unit catalog of the edge-ml server: every quantity with its + * units and the unit models expect (`defaultUnit`). No API key needed. + * @param {string} url - The url of the backend server + * @returns {Promise} + */ + function fetchUnits(url) { + return http.get(url, "/ml/units/"); + } + + /** + * Time-series as given by the app: a plain name, or `{ name, unit?, quantity? }`. + * @param {string | { name: string, unit?: string, quantity?: string }} ts + * @returns {{ name: string, unit?: string, quantity?: string }} + */ + function seriesSpec(ts) { + if (typeof ts === "string") return { name: ts }; + if (!ts || typeof ts.name !== "string" || !ts.name) { + throw new TypeError("A time-series needs a name, e.g. \"accX\" or { name: \"accX\", unit: \"g\", quantity: \"acceleration\" }"); + } + return ts; + } + + /** + * The time-series entry of a dataset init request. Plain names stay plain + * strings, so servers without unit support keep accepting them. + */ + function initEntry(ts) { + const { name, unit, quantity } = seriesSpec(ts); + if (!unit && !quantity) return name; + return { name, unit: unit || "", quantity: quantity || null }; + } + + /** + * The expected unit of every model input, from `manifest.input.units`. + * Exports made before units existed have none: every input is unknown. + * @returns {import("../index").InputUnit[]} + */ + function manifestUnits(manifest) { + const names = manifest.input.timeseries; + const byName = {}; + for (const u of manifest.input.units || []) byName[u.name] = u; + return names.map((name) => { + const u = byName[name] || {}; + return { name, quantity: u.quantity || null, unit: u.unit || "", conversions: u.conversions || {} }; + }); + } + + const isUnknown = (input) => !input.quantity && !input.unit; + + /** The catalog unit id of `unit` within `quantityId`, matching ids, labels and aliases. */ + function catalogUnitId(catalog, quantityId, unit) { + const quantity = catalog && (catalog.quantities || []).find((q) => q.id === quantityId); + if (!quantity) return null; + const match = quantity.units.find((u) => u.id === unit || u.label === unit || (u.aliases || []).includes(unit)); + return match ? match.id : null; + } + + /** + * Works out how to convert each input from the unit the app delivers to the + * unit the model expects. + * + * @param {import("../index").InputUnit[]} inputs - from `manifestUnits` + * @param {{ [sensor: string]: string }} sensorUnits - unit each sensor delivers; sensors the model does not use are ignored + * @param {import("../index").UnitCatalog} [catalog] - optional, to also accept unit labels and aliases (e.g. "°C") + * @returns {({ scale: number, offset: number } | null)[]} per input, null = pass values through + */ + function resolveConversions(inputs, sensorUnits, catalog) { + return inputs.map((input) => { + const unit = sensorUnits ? sensorUnits[input.name] : undefined; + if (unit === undefined || unit === null) return null; + if (typeof unit !== "string") throw new TypeError(`The unit of '${input.name}' must be a string`); + // trained before units existed, or on data of unknown unit: values go in as they are + if (isUnknown(input)) return null; + + let conv = input.conversions[unit]; + if (!conv && unit !== input.unit && input.quantity) { + const id = catalogUnitId(catalog, input.quantity, unit); + if (id) conv = input.conversions[id] || (id === input.unit ? { scale: 1, offset: 0 } : undefined); + } + if (!conv && unit === input.unit) conv = { scale: 1, offset: 0 }; + if (!conv) { + const accepted = Object.keys(input.conversions); + const expected = `${input.unit || "no unit"}${input.quantity ? ` (${input.quantity})` : ""}`; + throw new UnitError( + `'${input.name}' is declared in '${unit}', but the model expects ${expected} and cannot convert from '${unit}'. ` + + (accepted.length > 1 ? `Accepted units: ${accepted.join(", ")}.` : `Deliver it in ${input.unit || "the unit it was trained on"}.`) + ); + } + return conv.scale === 1 && conv.offset === 0 ? null : { scale: conv.scale, offset: conv.offset }; + }); + } + let runtime = null; async function getRuntime(ort) { @@ -324,8 +423,9 @@ * @param {any} session - ort.InferenceSession * @param {any} manifest - manifest.json of the export * @param {any} ort - the onnxruntime module + * @param {{ units?: { [sensor: string]: string }, catalog?: object }} [unitOptions] */ - constructor(session, manifest, ort) { + constructor(session, manifest, ort, { units, catalog } = {}) { if (!manifest || manifest.format !== "onnx") throw new Error("Not an edge-ml ONNX manifest"); this.session = session; this.manifest = manifest; @@ -337,6 +437,28 @@ /** @type {number} */ this.windowSize = manifest.window.size; this.buffer = new SampleBuffer(this.sensors, this.windowSize); + /** + * Unit the model expects for each input, with conversions from other units. + * Inputs of models trained before units existed have quantity null and unit "". + * @type {import("../../index").InputUnit[]} + */ + this.inputUnits = manifestUnits(manifest); + this._conversions = this.sensors.map(() => null); + this._catalog = catalog; + if (units) this.setUnits(units); + } + + /** + * Declares the unit each sensor delivers, e.g. `{ accX: "g" }`. Values are + * then converted to the unit the model expects before predicting. Sensors + * left out (and inputs of models without unit information) are passed through. + * Throws a UnitError if the model cannot convert from a declared unit. + * @param {{ [sensor: string]: string }} units + * @param {object} [catalog] - from `fetchUnits`, to also accept unit labels and aliases like "°C" + */ + setUnits(units, catalog = this._catalog) { + this._conversions = resolveConversions(this.inputUnits, units || {}, catalog); + this._catalog = catalog; } /** @@ -350,19 +472,21 @@ /** * @param {ArrayBuffer | Uint8Array} model - bytes of model.onnx * @param {object} manifest - parsed manifest.json - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {{ ort?: any, sessionOptions?: object, units?: { [sensor: string]: string }, catalog?: object }} [opts] */ - static async load(model, manifest, { ort, sessionOptions } = {}) { + static async load(model, manifest, { ort, sessionOptions, units, catalog } = {}) { + // check the units before creating the session, so a mistake fails fast + if (units) resolveConversions(manifestUnits(manifest), units, catalog); const rt = await getRuntime(ort); const session = await rt.InferenceSession.create(toBytes(model), sessionOptions); - return new OnnxPredictor(session, manifest, rt); + return new OnnxPredictor(session, manifest, rt, { units, catalog }); } /** * Loads model.onnx and manifest.json from URLs (e.g. an unzipped download from edge-ml). * @param {string} modelUrl * @param {string} manifestUrl - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromUrls(modelUrl, manifestUrl, opts) { const [model, manifest] = await Promise.all([ @@ -376,7 +500,7 @@ * Node.js only: loads model.onnx and manifest.json from disk. * @param {string} modelPath * @param {string} manifestPath - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromFiles(modelPath, manifestPath, opts) { const [model, manifest] = await Promise.all([readFile(), readFile()]); @@ -388,7 +512,7 @@ * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromServer(url, key, modelId, opts) { const [model, manifest] = await fetchOnnx(url, key, modelId); @@ -429,7 +553,7 @@ /** * Classifies one window. - * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order + * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order, in the declared units * @returns {Promise} */ async predictWindow(window) { @@ -437,7 +561,13 @@ if (window.length !== this.windowSize || window.some((row) => row.length !== n)) { throw new TypeError(`Window must have shape [${this.windowSize}][${n}]`); } - const data = Float32Array.from(window.flat()); + const data = new Float32Array(this.windowSize * n); + window.forEach((row, r) => + row.forEach((value, c) => { + const conv = this._conversions[c]; + data[r * n + c] = conv ? value * conv.scale + conv.offset : value; + }) + ); const input = new this.ort.Tensor("float32", data, [1, this.windowSize, n]); const outputs = await this.session.run({ [this.manifest.input.name]: input }); const scores = Array.from(outputs[this.manifest.output.name].data); @@ -459,7 +589,7 @@ * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); @@ -476,8 +606,10 @@ * Uploads a whole dataset to a specific project * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key (write access) - * @param {{ name: string, timeSeries: { name: string, data: [number, number][] }[], metaData?: { [key: string]: string }, labeling?: string }} dataset - * time-series data as [unix ms, value] pairs; labeling as "{labeling}_{label}" for the whole dataset + * @param {import("../index").DatasetUpload} dataset + * time-series data as [unix ms, value] pairs, each with an optional unit and quantity + * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as + * "{labeling}_{label}" for the whole dataset * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { @@ -487,7 +619,7 @@ const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map((ts) => ts.name), + timeSeries: dataset.timeSeries.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -508,7 +640,12 @@ * * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key + * @param {string} name - Name of the new dataset * @param {boolean} useDeviceTime - True if you want to use timestamps generated by the server + * @param {(string | { name: string, unit?: string, quantity?: string })[]} timeSeries - names, or + * objects with the unit and quantity of the series, e.g. { name: "accX", unit: "g", quantity: "acceleration" } + * @param {{ [key: string]: string }} [metaData] + * @param {string} [datasetLabel] - label for the whole dataset as "{labeling}_{label}" * @returns Function to upload single datapoints to one dataset inside a specific project */ async function datasetCollector( @@ -525,10 +662,15 @@ labeling = {"labelingName": datasetLabel.split("_")[0], "labelName": datasetLabel.split("_")[1]}; } + if (!Array.isArray(timeSeries) || timeSeries.length === 0) { + throw new Error("timeSeries must be a non-empty array"); + } + const seriesNames = timeSeries.map((ts) => seriesSpec(ts).name); + const data = await http.post(url, URLS.initDatasetIncrement + key, { name: name, - metaData: metaData, - timeSeries: timeSeries, + metaData: metaData || {}, + timeSeries: timeSeries.map(initEntry), labeling: labeling }); if (!data || !data.id) { @@ -540,7 +682,6 @@ var dataStore = { data: [] }; var lastChecked = Date.now(); var error = undefined; - var timeSeries = timeSeries; /** * Uploads a vlaue for a specific timestamp to a datasets timeSeries with name sensorName @@ -551,7 +692,7 @@ */ function addDataPoint(time, name, value) { - if (!timeSeries.includes(name)) { + if (!seriesNames.includes(name)) { throw Error("invalid time-series name") } @@ -638,6 +779,9 @@ // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + // units + fetchUnits: fetchUnits, + UnitError: UnitError, // training on the edge-ml server listPipelines: listPipelines, trainModel: trainModel, diff --git a/dist/index.js b/dist/index.js index 695ef71..7ce091f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -313,6 +313,105 @@ function fetchOnnx(url, key, modelId) { ]); } +/** + * Thrown when a sensor unit cannot be converted to the unit a model expects, + * so the app gets an error instead of wrong predictions. + */ +class UnitError extends Error {} + +/** + * Fetches the unit catalog of the edge-ml server: every quantity with its + * units and the unit models expect (`defaultUnit`). No API key needed. + * @param {string} url - The url of the backend server + * @returns {Promise} + */ +function fetchUnits(url) { + return http.get(url, "/ml/units/"); +} + +/** + * Time-series as given by the app: a plain name, or `{ name, unit?, quantity? }`. + * @param {string | { name: string, unit?: string, quantity?: string }} ts + * @returns {{ name: string, unit?: string, quantity?: string }} + */ +function seriesSpec(ts) { + if (typeof ts === "string") return { name: ts }; + if (!ts || typeof ts.name !== "string" || !ts.name) { + throw new TypeError("A time-series needs a name, e.g. \"accX\" or { name: \"accX\", unit: \"g\", quantity: \"acceleration\" }"); + } + return ts; +} + +/** + * The time-series entry of a dataset init request. Plain names stay plain + * strings, so servers without unit support keep accepting them. + */ +function initEntry(ts) { + const { name, unit, quantity } = seriesSpec(ts); + if (!unit && !quantity) return name; + return { name, unit: unit || "", quantity: quantity || null }; +} + +/** + * The expected unit of every model input, from `manifest.input.units`. + * Exports made before units existed have none: every input is unknown. + * @returns {import("../index").InputUnit[]} + */ +function manifestUnits(manifest) { + const names = manifest.input.timeseries; + const byName = {}; + for (const u of manifest.input.units || []) byName[u.name] = u; + return names.map((name) => { + const u = byName[name] || {}; + return { name, quantity: u.quantity || null, unit: u.unit || "", conversions: u.conversions || {} }; + }); +} + +const isUnknown = (input) => !input.quantity && !input.unit; + +/** The catalog unit id of `unit` within `quantityId`, matching ids, labels and aliases. */ +function catalogUnitId(catalog, quantityId, unit) { + const quantity = catalog && (catalog.quantities || []).find((q) => q.id === quantityId); + if (!quantity) return null; + const match = quantity.units.find((u) => u.id === unit || u.label === unit || (u.aliases || []).includes(unit)); + return match ? match.id : null; +} + +/** + * Works out how to convert each input from the unit the app delivers to the + * unit the model expects. + * + * @param {import("../index").InputUnit[]} inputs - from `manifestUnits` + * @param {{ [sensor: string]: string }} sensorUnits - unit each sensor delivers; sensors the model does not use are ignored + * @param {import("../index").UnitCatalog} [catalog] - optional, to also accept unit labels and aliases (e.g. "°C") + * @returns {({ scale: number, offset: number } | null)[]} per input, null = pass values through + */ +function resolveConversions(inputs, sensorUnits, catalog) { + return inputs.map((input) => { + const unit = sensorUnits ? sensorUnits[input.name] : undefined; + if (unit === undefined || unit === null) return null; + if (typeof unit !== "string") throw new TypeError(`The unit of '${input.name}' must be a string`); + // trained before units existed, or on data of unknown unit: values go in as they are + if (isUnknown(input)) return null; + + let conv = input.conversions[unit]; + if (!conv && unit !== input.unit && input.quantity) { + const id = catalogUnitId(catalog, input.quantity, unit); + if (id) conv = input.conversions[id] || (id === input.unit ? { scale: 1, offset: 0 } : undefined); + } + if (!conv && unit === input.unit) conv = { scale: 1, offset: 0 }; + if (!conv) { + const accepted = Object.keys(input.conversions); + const expected = `${input.unit || "no unit"}${input.quantity ? ` (${input.quantity})` : ""}`; + throw new UnitError( + `'${input.name}' is declared in '${unit}', but the model expects ${expected} and cannot convert from '${unit}'. ` + + (accepted.length > 1 ? `Accepted units: ${accepted.join(", ")}.` : `Deliver it in ${input.unit || "the unit it was trained on"}.`) + ); + } + return conv.scale === 1 && conv.offset === 0 ? null : { scale: conv.scale, offset: conv.offset }; + }); +} + let runtime = null; async function getRuntime(ort) { @@ -342,8 +441,9 @@ class OnnxPredictor { * @param {any} session - ort.InferenceSession * @param {any} manifest - manifest.json of the export * @param {any} ort - the onnxruntime module + * @param {{ units?: { [sensor: string]: string }, catalog?: object }} [unitOptions] */ - constructor(session, manifest, ort) { + constructor(session, manifest, ort, { units, catalog } = {}) { if (!manifest || manifest.format !== "onnx") throw new Error("Not an edge-ml ONNX manifest"); this.session = session; this.manifest = manifest; @@ -355,6 +455,28 @@ class OnnxPredictor { /** @type {number} */ this.windowSize = manifest.window.size; this.buffer = new SampleBuffer(this.sensors, this.windowSize); + /** + * Unit the model expects for each input, with conversions from other units. + * Inputs of models trained before units existed have quantity null and unit "". + * @type {import("../../index").InputUnit[]} + */ + this.inputUnits = manifestUnits(manifest); + this._conversions = this.sensors.map(() => null); + this._catalog = catalog; + if (units) this.setUnits(units); + } + + /** + * Declares the unit each sensor delivers, e.g. `{ accX: "g" }`. Values are + * then converted to the unit the model expects before predicting. Sensors + * left out (and inputs of models without unit information) are passed through. + * Throws a UnitError if the model cannot convert from a declared unit. + * @param {{ [sensor: string]: string }} units + * @param {object} [catalog] - from `fetchUnits`, to also accept unit labels and aliases like "°C" + */ + setUnits(units, catalog = this._catalog) { + this._conversions = resolveConversions(this.inputUnits, units || {}, catalog); + this._catalog = catalog; } /** @@ -368,19 +490,21 @@ class OnnxPredictor { /** * @param {ArrayBuffer | Uint8Array} model - bytes of model.onnx * @param {object} manifest - parsed manifest.json - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {{ ort?: any, sessionOptions?: object, units?: { [sensor: string]: string }, catalog?: object }} [opts] */ - static async load(model, manifest, { ort, sessionOptions } = {}) { + static async load(model, manifest, { ort, sessionOptions, units, catalog } = {}) { + // check the units before creating the session, so a mistake fails fast + if (units) resolveConversions(manifestUnits(manifest), units, catalog); const rt = await getRuntime(ort); const session = await rt.InferenceSession.create(toBytes(model), sessionOptions); - return new OnnxPredictor(session, manifest, rt); + return new OnnxPredictor(session, manifest, rt, { units, catalog }); } /** * Loads model.onnx and manifest.json from URLs (e.g. an unzipped download from edge-ml). * @param {string} modelUrl * @param {string} manifestUrl - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromUrls(modelUrl, manifestUrl, opts) { const [model, manifest] = await Promise.all([ @@ -394,7 +518,7 @@ class OnnxPredictor { * Node.js only: loads model.onnx and manifest.json from disk. * @param {string} modelPath * @param {string} manifestPath - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromFiles(modelPath, manifestPath, opts) { const [model, manifest] = await Promise.all([readFile(modelPath), readFile(manifestPath)]); @@ -406,7 +530,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromServer(url, key, modelId, opts) { const [model, manifest] = await fetchOnnx(url, key, modelId); @@ -447,7 +571,7 @@ class OnnxPredictor { /** * Classifies one window. - * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order + * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order, in the declared units * @returns {Promise} */ async predictWindow(window) { @@ -455,7 +579,13 @@ class OnnxPredictor { if (window.length !== this.windowSize || window.some((row) => row.length !== n)) { throw new TypeError(`Window must have shape [${this.windowSize}][${n}]`); } - const data = Float32Array.from(window.flat()); + const data = new Float32Array(this.windowSize * n); + window.forEach((row, r) => + row.forEach((value, c) => { + const conv = this._conversions[c]; + data[r * n + c] = conv ? value * conv.scale + conv.offset : value; + }) + ); const input = new this.ort.Tensor("float32", data, [1, this.windowSize, n]); const outputs = await this.session.run({ [this.manifest.input.name]: input }); const scores = Array.from(outputs[this.manifest.output.name].data); @@ -477,7 +607,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); @@ -494,8 +624,10 @@ const URLS = { * Uploads a whole dataset to a specific project * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key (write access) - * @param {{ name: string, timeSeries: { name: string, data: [number, number][] }[], metaData?: { [key: string]: string }, labeling?: string }} dataset - * time-series data as [unix ms, value] pairs; labeling as "{labeling}_{label}" for the whole dataset + * @param {import("../index").DatasetUpload} dataset + * time-series data as [unix ms, value] pairs, each with an optional unit and quantity + * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as + * "{labeling}_{label}" for the whole dataset * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { @@ -505,7 +637,7 @@ async function sendDataset(url, key, dataset) { const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map((ts) => ts.name), + timeSeries: dataset.timeSeries.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -526,7 +658,12 @@ async function sendDataset(url, key, dataset) { * * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key + * @param {string} name - Name of the new dataset * @param {boolean} useDeviceTime - True if you want to use timestamps generated by the server + * @param {(string | { name: string, unit?: string, quantity?: string })[]} timeSeries - names, or + * objects with the unit and quantity of the series, e.g. { name: "accX", unit: "g", quantity: "acceleration" } + * @param {{ [key: string]: string }} [metaData] + * @param {string} [datasetLabel] - label for the whole dataset as "{labeling}_{label}" * @returns Function to upload single datapoints to one dataset inside a specific project */ async function datasetCollector( @@ -543,10 +680,15 @@ async function datasetCollector( labeling = {"labelingName": datasetLabel.split("_")[0], "labelName": datasetLabel.split("_")[1]}; } + if (!Array.isArray(timeSeries) || timeSeries.length === 0) { + throw new Error("timeSeries must be a non-empty array"); + } + const seriesNames = timeSeries.map((ts) => seriesSpec(ts).name); + const data = await http.post(url, URLS.initDatasetIncrement + key, { name: name, - metaData: metaData, - timeSeries: timeSeries, + metaData: metaData || {}, + timeSeries: timeSeries.map(initEntry), labeling: labeling }); if (!data || !data.id) { @@ -558,7 +700,6 @@ async function datasetCollector( var dataStore = { data: [] }; var lastChecked = Date.now(); var error = undefined; - var timeSeries = timeSeries; /** * Uploads a vlaue for a specific timestamp to a datasets timeSeries with name sensorName @@ -569,7 +710,7 @@ async function datasetCollector( */ function addDataPoint(time, name, value) { - if (!timeSeries.includes(name)) { + if (!seriesNames.includes(name)) { throw Error("invalid time-series name") } @@ -656,6 +797,9 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + // units + fetchUnits: fetchUnits, + UnitError: UnitError, // training on the edge-ml server listPipelines: listPipelines, trainModel: trainModel, diff --git a/dist/index.mjs b/dist/index.mjs index 10324c5..704a41c 100644 --- a/dist/index.mjs +++ b/dist/index.mjs @@ -294,6 +294,105 @@ function fetchOnnx(url, key, modelId) { ]); } +/** + * Thrown when a sensor unit cannot be converted to the unit a model expects, + * so the app gets an error instead of wrong predictions. + */ +class UnitError extends Error {} + +/** + * Fetches the unit catalog of the edge-ml server: every quantity with its + * units and the unit models expect (`defaultUnit`). No API key needed. + * @param {string} url - The url of the backend server + * @returns {Promise} + */ +function fetchUnits(url) { + return http.get(url, "/ml/units/"); +} + +/** + * Time-series as given by the app: a plain name, or `{ name, unit?, quantity? }`. + * @param {string | { name: string, unit?: string, quantity?: string }} ts + * @returns {{ name: string, unit?: string, quantity?: string }} + */ +function seriesSpec(ts) { + if (typeof ts === "string") return { name: ts }; + if (!ts || typeof ts.name !== "string" || !ts.name) { + throw new TypeError("A time-series needs a name, e.g. \"accX\" or { name: \"accX\", unit: \"g\", quantity: \"acceleration\" }"); + } + return ts; +} + +/** + * The time-series entry of a dataset init request. Plain names stay plain + * strings, so servers without unit support keep accepting them. + */ +function initEntry(ts) { + const { name, unit, quantity } = seriesSpec(ts); + if (!unit && !quantity) return name; + return { name, unit: unit || "", quantity: quantity || null }; +} + +/** + * The expected unit of every model input, from `manifest.input.units`. + * Exports made before units existed have none: every input is unknown. + * @returns {import("../index").InputUnit[]} + */ +function manifestUnits(manifest) { + const names = manifest.input.timeseries; + const byName = {}; + for (const u of manifest.input.units || []) byName[u.name] = u; + return names.map((name) => { + const u = byName[name] || {}; + return { name, quantity: u.quantity || null, unit: u.unit || "", conversions: u.conversions || {} }; + }); +} + +const isUnknown = (input) => !input.quantity && !input.unit; + +/** The catalog unit id of `unit` within `quantityId`, matching ids, labels and aliases. */ +function catalogUnitId(catalog, quantityId, unit) { + const quantity = catalog && (catalog.quantities || []).find((q) => q.id === quantityId); + if (!quantity) return null; + const match = quantity.units.find((u) => u.id === unit || u.label === unit || (u.aliases || []).includes(unit)); + return match ? match.id : null; +} + +/** + * Works out how to convert each input from the unit the app delivers to the + * unit the model expects. + * + * @param {import("../index").InputUnit[]} inputs - from `manifestUnits` + * @param {{ [sensor: string]: string }} sensorUnits - unit each sensor delivers; sensors the model does not use are ignored + * @param {import("../index").UnitCatalog} [catalog] - optional, to also accept unit labels and aliases (e.g. "°C") + * @returns {({ scale: number, offset: number } | null)[]} per input, null = pass values through + */ +function resolveConversions(inputs, sensorUnits, catalog) { + return inputs.map((input) => { + const unit = sensorUnits ? sensorUnits[input.name] : undefined; + if (unit === undefined || unit === null) return null; + if (typeof unit !== "string") throw new TypeError(`The unit of '${input.name}' must be a string`); + // trained before units existed, or on data of unknown unit: values go in as they are + if (isUnknown(input)) return null; + + let conv = input.conversions[unit]; + if (!conv && unit !== input.unit && input.quantity) { + const id = catalogUnitId(catalog, input.quantity, unit); + if (id) conv = input.conversions[id] || (id === input.unit ? { scale: 1, offset: 0 } : undefined); + } + if (!conv && unit === input.unit) conv = { scale: 1, offset: 0 }; + if (!conv) { + const accepted = Object.keys(input.conversions); + const expected = `${input.unit || "no unit"}${input.quantity ? ` (${input.quantity})` : ""}`; + throw new UnitError( + `'${input.name}' is declared in '${unit}', but the model expects ${expected} and cannot convert from '${unit}'. ` + + (accepted.length > 1 ? `Accepted units: ${accepted.join(", ")}.` : `Deliver it in ${input.unit || "the unit it was trained on"}.`) + ); + } + return conv.scale === 1 && conv.offset === 0 ? null : { scale: conv.scale, offset: conv.offset }; + }); +} + let runtime = null; async function getRuntime(ort) { @@ -323,8 +422,9 @@ class OnnxPredictor { * @param {any} session - ort.InferenceSession * @param {any} manifest - manifest.json of the export * @param {any} ort - the onnxruntime module + * @param {{ units?: { [sensor: string]: string }, catalog?: object }} [unitOptions] */ - constructor(session, manifest, ort) { + constructor(session, manifest, ort, { units, catalog } = {}) { if (!manifest || manifest.format !== "onnx") throw new Error("Not an edge-ml ONNX manifest"); this.session = session; this.manifest = manifest; @@ -336,6 +436,28 @@ class OnnxPredictor { /** @type {number} */ this.windowSize = manifest.window.size; this.buffer = new SampleBuffer(this.sensors, this.windowSize); + /** + * Unit the model expects for each input, with conversions from other units. + * Inputs of models trained before units existed have quantity null and unit "". + * @type {import("../../index").InputUnit[]} + */ + this.inputUnits = manifestUnits(manifest); + this._conversions = this.sensors.map(() => null); + this._catalog = catalog; + if (units) this.setUnits(units); + } + + /** + * Declares the unit each sensor delivers, e.g. `{ accX: "g" }`. Values are + * then converted to the unit the model expects before predicting. Sensors + * left out (and inputs of models without unit information) are passed through. + * Throws a UnitError if the model cannot convert from a declared unit. + * @param {{ [sensor: string]: string }} units + * @param {object} [catalog] - from `fetchUnits`, to also accept unit labels and aliases like "°C" + */ + setUnits(units, catalog = this._catalog) { + this._conversions = resolveConversions(this.inputUnits, units || {}, catalog); + this._catalog = catalog; } /** @@ -349,19 +471,21 @@ class OnnxPredictor { /** * @param {ArrayBuffer | Uint8Array} model - bytes of model.onnx * @param {object} manifest - parsed manifest.json - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {{ ort?: any, sessionOptions?: object, units?: { [sensor: string]: string }, catalog?: object }} [opts] */ - static async load(model, manifest, { ort, sessionOptions } = {}) { + static async load(model, manifest, { ort, sessionOptions, units, catalog } = {}) { + // check the units before creating the session, so a mistake fails fast + if (units) resolveConversions(manifestUnits(manifest), units, catalog); const rt = await getRuntime(ort); const session = await rt.InferenceSession.create(toBytes(model), sessionOptions); - return new OnnxPredictor(session, manifest, rt); + return new OnnxPredictor(session, manifest, rt, { units, catalog }); } /** * Loads model.onnx and manifest.json from URLs (e.g. an unzipped download from edge-ml). * @param {string} modelUrl * @param {string} manifestUrl - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromUrls(modelUrl, manifestUrl, opts) { const [model, manifest] = await Promise.all([ @@ -375,7 +499,7 @@ class OnnxPredictor { * Node.js only: loads model.onnx and manifest.json from disk. * @param {string} modelPath * @param {string} manifestPath - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromFiles(modelPath, manifestPath, opts) { const [model, manifest] = await Promise.all([readFile(modelPath), readFile(manifestPath)]); @@ -387,7 +511,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromServer(url, key, modelId, opts) { const [model, manifest] = await fetchOnnx(url, key, modelId); @@ -428,7 +552,7 @@ class OnnxPredictor { /** * Classifies one window. - * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order + * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order, in the declared units * @returns {Promise} */ async predictWindow(window) { @@ -436,7 +560,13 @@ class OnnxPredictor { if (window.length !== this.windowSize || window.some((row) => row.length !== n)) { throw new TypeError(`Window must have shape [${this.windowSize}][${n}]`); } - const data = Float32Array.from(window.flat()); + const data = new Float32Array(this.windowSize * n); + window.forEach((row, r) => + row.forEach((value, c) => { + const conv = this._conversions[c]; + data[r * n + c] = conv ? value * conv.scale + conv.offset : value; + }) + ); const input = new this.ort.Tensor("float32", data, [1, this.windowSize, n]); const outputs = await this.session.run({ [this.manifest.input.name]: input }); const scores = Array.from(outputs[this.manifest.output.name].data); @@ -458,7 +588,7 @@ class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); @@ -475,8 +605,10 @@ const URLS = { * Uploads a whole dataset to a specific project * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key (write access) - * @param {{ name: string, timeSeries: { name: string, data: [number, number][] }[], metaData?: { [key: string]: string }, labeling?: string }} dataset - * time-series data as [unix ms, value] pairs; labeling as "{labeling}_{label}" for the whole dataset + * @param {import("../index").DatasetUpload} dataset + * time-series data as [unix ms, value] pairs, each with an optional unit and quantity + * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as + * "{labeling}_{label}" for the whole dataset * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { @@ -486,7 +618,7 @@ async function sendDataset(url, key, dataset) { const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map((ts) => ts.name), + timeSeries: dataset.timeSeries.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -507,7 +639,12 @@ async function sendDataset(url, key, dataset) { * * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key + * @param {string} name - Name of the new dataset * @param {boolean} useDeviceTime - True if you want to use timestamps generated by the server + * @param {(string | { name: string, unit?: string, quantity?: string })[]} timeSeries - names, or + * objects with the unit and quantity of the series, e.g. { name: "accX", unit: "g", quantity: "acceleration" } + * @param {{ [key: string]: string }} [metaData] + * @param {string} [datasetLabel] - label for the whole dataset as "{labeling}_{label}" * @returns Function to upload single datapoints to one dataset inside a specific project */ async function datasetCollector( @@ -524,10 +661,15 @@ async function datasetCollector( labeling = {"labelingName": datasetLabel.split("_")[0], "labelName": datasetLabel.split("_")[1]}; } + if (!Array.isArray(timeSeries) || timeSeries.length === 0) { + throw new Error("timeSeries must be a non-empty array"); + } + const seriesNames = timeSeries.map((ts) => seriesSpec(ts).name); + const data = await http.post(url, URLS.initDatasetIncrement + key, { name: name, - metaData: metaData, - timeSeries: timeSeries, + metaData: metaData || {}, + timeSeries: timeSeries.map(initEntry), labeling: labeling }); if (!data || !data.id) { @@ -539,7 +681,6 @@ async function datasetCollector( var dataStore = { data: [] }; var lastChecked = Date.now(); var error = undefined; - var timeSeries = timeSeries; /** * Uploads a vlaue for a specific timestamp to a datasets timeSeries with name sensorName @@ -550,7 +691,7 @@ async function datasetCollector( */ function addDataPoint(time, name, value) { - if (!timeSeries.includes(name)) { + if (!seriesNames.includes(name)) { throw Error("invalid time-series name") } @@ -637,6 +778,9 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + // units + fetchUnits: fetchUnits, + UnitError: UnitError, // training on the edge-ml server listPipelines: listPipelines, trainModel: trainModel, @@ -650,4 +794,4 @@ const edgeML = { PredictorError: PredictorError, }; -export { OnnxPredictor, PredictorError, datasetCollector, edgeML as default, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; +export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, fetchUnits, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; diff --git a/examples/example-train-and-predict-web/index.html b/examples/example-train-and-predict-web/index.html index 431b0a9..79ad525 100644 --- a/examples/example-train-and-predict-web/index.html +++ b/examples/example-train-and-predict-web/index.html @@ -40,6 +40,9 @@

3. Predict

const $ = (id) => document.getElementById(id); const log = (msg) => ($("log").textContent = msg + "\n" + $("log").textContent); const SENSORS = ["accX", "accY", "accZ"]; + // devicemotion reports acceleration in m/s² + const UNITS = { accX: "m/s^2", accY: "m/s^2", accZ: "m/s^2" }; + const SERIES = SENSORS.map((name) => ({ name, unit: UNITS[name], quantity: "acceleration" })); let model = null; // devicemotion needs a permission prompt on iOS @@ -58,7 +61,7 @@

3. Predict

$("record").onclick = async () => { const label = $("label").value; const collector = await edgeML.datasetCollector( - $("url").value, $("key").value, `${label}-${Date.now()}`, true, SENSORS, {}, `${$("labeling").value}_${label}` + $("url").value, $("key").value, `${label}-${Date.now()}`, true, SERIES, {}, `${$("labeling").value}_${label}` ); const stop = await onMotion((s) => SENSORS.forEach((n) => collector.addDataPoint(n, s[n]))); log(`recording '${label}'…`); @@ -85,7 +88,7 @@

3. Predict

$("predict").onclick = async () => { if (!model) return log("train a model first"); - const predictor = await edgeML.loadModel($("url").value, $("key").value, model.id); + const predictor = await edgeML.loadModel($("url").value, $("key").value, model.id, { units: UNITS }); await onMotion((s) => predictor.addSample(s)); setInterval(async () => { try { diff --git a/index.d.ts b/index.d.ts index 15c9f9c..b091716 100644 --- a/index.d.ts +++ b/index.d.ts @@ -16,23 +16,37 @@ export interface DatasetCollectorOwnTime { onComplete(): Promise; } +/** + * Unit of a time-series. `quantity` is a quantity id of the unit catalog + * (see `fetchUnits`), e.g. "acceleration", or a label of your own; `unit` is + * a unit id of that quantity, e.g. "g". Training converts series of catalog + * quantities to the quantity's default unit. + */ +export interface SeriesUnit { + unit?: string; + quantity?: string; +} + +/** A time-series name, or a name with its unit, e.g. `{ name: "accX", unit: "g", quantity: "acceleration" }`. */ +export type TimeSeriesSpec = string | ({ name: string } & SeriesUnit); + /** * Creates a dataset and returns a collector that uploads datapoints incrementally. * @param datasetLabel label for the whole dataset as "{labeling}_{label}" */ export function datasetCollector( - url: string, key: string, name: string, useDeviceTime: true, timeSeries: string[], + url: string, key: string, name: string, useDeviceTime: true, timeSeries: TimeSeriesSpec[], metaData?: Record, datasetLabel?: string ): Promise; export function datasetCollector( - url: string, key: string, name: string, useDeviceTime: false, timeSeries: string[], + url: string, key: string, name: string, useDeviceTime: false, timeSeries: TimeSeriesSpec[], metaData?: Record, datasetLabel?: string ): Promise; export interface DatasetUpload { name: string; - /** data as [unix ms, value] pairs */ - timeSeries: { name: string; data: [number, number][] }[]; + /** data as [unix ms, value] pairs, optionally with the unit of the series */ + timeSeries: ({ name: string; data: [number, number][] } & SeriesUnit)[]; metaData?: Record; /** label for the whole dataset as "{labeling}_{label}" */ labeling?: string; @@ -41,6 +55,47 @@ export interface DatasetUpload { /** Uploads a whole dataset. Resolves to the id of the new dataset. */ export function sendDataset(url: string, key: string, dataset: DatasetUpload): Promise; +// ---------------------------------------------------------------- units + +export interface CatalogUnit { + id: string; + label: string; + /** affine map to the quantity's base unit: base = value * scale + offset */ + scale: number; + offset: number; + aliases: string[]; +} + +export interface CatalogQuantity { + id: string; + label: string; + /** the unit training converts to, and models expect */ + defaultUnit: string; + units: CatalogUnit[]; +} + +export interface UnitCatalog { + version: number; + quantities: CatalogQuantity[]; +} + +/** The unit a model input expects, and how to convert other units to it. */ +export interface InputUnit { + name: string; + /** catalog quantity id or custom label; null when unknown (e.g. models trained before units existed) */ + quantity: string | null; + /** expected unit; "" when unknown */ + unit: string; + /** per catalog unit id: expected = value * scale + offset */ + conversions: Record; +} + +/** The unit catalog of the server: quantities, their units and default units. No API key needed. */ +export function fetchUnits(url: string): Promise; + +/** A declared sensor unit that the model cannot convert from. */ +export class UnitError extends Error {} + // ---------------------------------------------------------------- training export interface PipelineStepOptions { @@ -90,6 +145,8 @@ export interface ModelInfo { formats: string[]; labels: string[]; timeSeries: string[]; + /** the unit each input expects (quantity null and unit "" for models trained before units existed) */ + inputUnits?: InputUnit[]; samplingRate: number | null; metrics: Record | null; } @@ -132,6 +189,13 @@ export interface LoadOptions { ort?: any; /** passed to ort.InferenceSession.create */ sessionOptions?: Record; + /** + * Unit each sensor delivers, e.g. `{ accX: "g" }`: values are converted to + * the unit the model expects. Throws a UnitError if the model cannot convert. + */ + units?: Record; + /** unit catalog (`fetchUnits`), to also accept unit labels and aliases such as "°C" in `units` */ + catalog?: UnitCatalog; } export class PredictorError extends Error {} @@ -145,6 +209,8 @@ export class OnnxPredictor { /** samples per window */ readonly windowSize: number; readonly manifest: any; + /** unit each input expects, from the manifest */ + readonly inputUnits: InputUnit[]; /** Sets the onnxruntime module (needed with bundlers in the browser). */ static setRuntime(ort: any): void; @@ -159,9 +225,14 @@ export class OnnxPredictor { /** One value per sensor with a shared timestamp. */ addSample(values: number[] | Record, time?: number): void; reset(): void; + /** + * Declares the unit each sensor delivers (replacing earlier declarations). + * Throws a UnitError if the model cannot convert from a declared unit. + */ + setUnits(units: Record, catalog?: UnitCatalog): void; /** Classifies the latest window; throws PredictorError until enough samples are buffered. */ predict(): Promise; - /** Classifies one window of `windowSize` rows in `sensors` order. */ + /** Classifies one window of `windowSize` rows in `sensors` order (in the declared units). */ predictWindow(window: number[][]): Promise; } @@ -172,6 +243,8 @@ export function loadModel(url: string, key: string, modelId: string, opts?: Load declare const edgeML: { datasetCollector: typeof datasetCollector; sendDataset: typeof sendDataset; + fetchUnits: typeof fetchUnits; + UnitError: typeof UnitError; listPipelines: typeof listPipelines; trainModel: typeof trainModel; startTraining: typeof startTraining; diff --git a/src/index.js b/src/index.js index 3105193..738bc02 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,7 @@ import { http } from "./http"; import { PredictorError } from "./errors"; import { OnnxPredictor, loadModel } from "./onnx/OnnxPredictor"; +import { UnitError, fetchUnits, initEntry, seriesSpec } from "./units"; import { listPipelines, startTraining, trainModel, waitForModel, getModel, listModels } from "./training"; const UPLOAD_INTERVAL = 5 * 1000; @@ -14,8 +15,10 @@ const URLS = { * Uploads a whole dataset to a specific project * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key (write access) - * @param {{ name: string, timeSeries: { name: string, data: [number, number][] }[], metaData?: { [key: string]: string }, labeling?: string }} dataset - * time-series data as [unix ms, value] pairs; labeling as "{labeling}_{label}" for the whole dataset + * @param {import("../index").DatasetUpload} dataset + * time-series data as [unix ms, value] pairs, each with an optional unit and quantity + * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as + * "{labeling}_{label}" for the whole dataset * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { @@ -25,7 +28,7 @@ async function sendDataset(url, key, dataset) { const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map((ts) => ts.name), + timeSeries: dataset.timeSeries.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -46,7 +49,12 @@ async function sendDataset(url, key, dataset) { * * @param {string} url - The url of the backend server * @param {string} key - The Device-Api-Key + * @param {string} name - Name of the new dataset * @param {boolean} useDeviceTime - True if you want to use timestamps generated by the server + * @param {(string | { name: string, unit?: string, quantity?: string })[]} timeSeries - names, or + * objects with the unit and quantity of the series, e.g. { name: "accX", unit: "g", quantity: "acceleration" } + * @param {{ [key: string]: string }} [metaData] + * @param {string} [datasetLabel] - label for the whole dataset as "{labeling}_{label}" * @returns Function to upload single datapoints to one dataset inside a specific project */ async function datasetCollector( @@ -63,10 +71,15 @@ async function datasetCollector( labeling = {"labelingName": datasetLabel.split("_")[0], "labelName": datasetLabel.split("_")[1]} } + if (!Array.isArray(timeSeries) || timeSeries.length === 0) { + throw new Error("timeSeries must be a non-empty array"); + } + const seriesNames = timeSeries.map((ts) => seriesSpec(ts).name); + const data = await http.post(url, URLS.initDatasetIncrement + key, { name: name, - metaData: metaData, - timeSeries: timeSeries, + metaData: metaData || {}, + timeSeries: timeSeries.map(initEntry), labeling: labeling }); if (!data || !data.id) { @@ -78,7 +91,6 @@ async function datasetCollector( var dataStore = { data: [] }; var lastChecked = Date.now() var error = undefined; - var timeSeries = timeSeries; /** * Uploads a vlaue for a specific timestamp to a datasets timeSeries with name sensorName @@ -89,7 +101,7 @@ async function datasetCollector( */ function addDataPoint(time, name, value) { - if (!timeSeries.includes(name)) { + if (!seriesNames.includes(name)) { throw Error("invalid time-series name") } @@ -176,6 +188,9 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + // units + fetchUnits: fetchUnits, + UnitError: UnitError, // training on the edge-ml server listPipelines: listPipelines, trainModel: trainModel, @@ -193,6 +208,8 @@ export default edgeML; export { datasetCollector, sendDataset, + fetchUnits, + UnitError, listPipelines, trainModel, startTraining, diff --git a/src/onnx/OnnxPredictor.js b/src/onnx/OnnxPredictor.js index e314f6a..955713e 100644 --- a/src/onnx/OnnxPredictor.js +++ b/src/onnx/OnnxPredictor.js @@ -1,6 +1,7 @@ import { SampleBuffer } from "./SampleBuffer"; import { loadRuntime, readFile, RUNTIME_HINT } from "./platform.node"; import { fetchOnnx } from "../training"; +import { manifestUnits, resolveConversions } from "../units"; let runtime = null; @@ -31,8 +32,9 @@ export class OnnxPredictor { * @param {any} session - ort.InferenceSession * @param {any} manifest - manifest.json of the export * @param {any} ort - the onnxruntime module + * @param {{ units?: { [sensor: string]: string }, catalog?: object }} [unitOptions] */ - constructor(session, manifest, ort) { + constructor(session, manifest, ort, { units, catalog } = {}) { if (!manifest || manifest.format !== "onnx") throw new Error("Not an edge-ml ONNX manifest"); this.session = session; this.manifest = manifest; @@ -44,6 +46,28 @@ export class OnnxPredictor { /** @type {number} */ this.windowSize = manifest.window.size; this.buffer = new SampleBuffer(this.sensors, this.windowSize); + /** + * Unit the model expects for each input, with conversions from other units. + * Inputs of models trained before units existed have quantity null and unit "". + * @type {import("../../index").InputUnit[]} + */ + this.inputUnits = manifestUnits(manifest); + this._conversions = this.sensors.map(() => null); + this._catalog = catalog; + if (units) this.setUnits(units); + } + + /** + * Declares the unit each sensor delivers, e.g. `{ accX: "g" }`. Values are + * then converted to the unit the model expects before predicting. Sensors + * left out (and inputs of models without unit information) are passed through. + * Throws a UnitError if the model cannot convert from a declared unit. + * @param {{ [sensor: string]: string }} units + * @param {object} [catalog] - from `fetchUnits`, to also accept unit labels and aliases like "°C" + */ + setUnits(units, catalog = this._catalog) { + this._conversions = resolveConversions(this.inputUnits, units || {}, catalog); + this._catalog = catalog; } /** @@ -57,19 +81,21 @@ export class OnnxPredictor { /** * @param {ArrayBuffer | Uint8Array} model - bytes of model.onnx * @param {object} manifest - parsed manifest.json - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {{ ort?: any, sessionOptions?: object, units?: { [sensor: string]: string }, catalog?: object }} [opts] */ - static async load(model, manifest, { ort, sessionOptions } = {}) { + static async load(model, manifest, { ort, sessionOptions, units, catalog } = {}) { + // check the units before creating the session, so a mistake fails fast + if (units) resolveConversions(manifestUnits(manifest), units, catalog); const rt = await getRuntime(ort); const session = await rt.InferenceSession.create(toBytes(model), sessionOptions); - return new OnnxPredictor(session, manifest, rt); + return new OnnxPredictor(session, manifest, rt, { units, catalog }); } /** * Loads model.onnx and manifest.json from URLs (e.g. an unzipped download from edge-ml). * @param {string} modelUrl * @param {string} manifestUrl - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromUrls(modelUrl, manifestUrl, opts) { const [model, manifest] = await Promise.all([ @@ -83,7 +109,7 @@ export class OnnxPredictor { * Node.js only: loads model.onnx and manifest.json from disk. * @param {string} modelPath * @param {string} manifestPath - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromFiles(modelPath, manifestPath, opts) { const [model, manifest] = await Promise.all([readFile(modelPath), readFile(manifestPath)]); @@ -95,7 +121,7 @@ export class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ static async fromServer(url, key, modelId, opts) { const [model, manifest] = await fetchOnnx(url, key, modelId); @@ -136,7 +162,7 @@ export class OnnxPredictor { /** * Classifies one window. - * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order + * @param {number[][]} window - `windowSize` rows of values in `predictor.sensors` order, in the declared units * @returns {Promise} */ async predictWindow(window) { @@ -144,7 +170,13 @@ export class OnnxPredictor { if (window.length !== this.windowSize || window.some((row) => row.length !== n)) { throw new TypeError(`Window must have shape [${this.windowSize}][${n}]`); } - const data = Float32Array.from(window.flat()); + const data = new Float32Array(this.windowSize * n); + window.forEach((row, r) => + row.forEach((value, c) => { + const conv = this._conversions[c]; + data[r * n + c] = conv ? value * conv.scale + conv.offset : value; + }) + ); const input = new this.ort.Tensor("float32", data, [1, this.windowSize, n]); const outputs = await this.session.run({ [this.manifest.input.name]: input }); const scores = Array.from(outputs[this.manifest.output.name].data); @@ -166,7 +198,7 @@ export class OnnxPredictor { * @param {string} url - The url of the backend server * @param {string} key - A Device-Api-Key (read or write) * @param {string} modelId - * @param {{ ort?: any, sessionOptions?: object }} [opts] + * @param {import("../../index").LoadOptions} [opts] */ export function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); diff --git a/src/units.js b/src/units.js new file mode 100644 index 0000000..cfdaf7b --- /dev/null +++ b/src/units.js @@ -0,0 +1,100 @@ +import { http } from "./http"; + +/** + * Thrown when a sensor unit cannot be converted to the unit a model expects, + * so the app gets an error instead of wrong predictions. + */ +export class UnitError extends Error {} + +/** + * Fetches the unit catalog of the edge-ml server: every quantity with its + * units and the unit models expect (`defaultUnit`). No API key needed. + * @param {string} url - The url of the backend server + * @returns {Promise} + */ +export function fetchUnits(url) { + return http.get(url, "/ml/units/"); +} + +/** + * Time-series as given by the app: a plain name, or `{ name, unit?, quantity? }`. + * @param {string | { name: string, unit?: string, quantity?: string }} ts + * @returns {{ name: string, unit?: string, quantity?: string }} + */ +export function seriesSpec(ts) { + if (typeof ts === "string") return { name: ts }; + if (!ts || typeof ts.name !== "string" || !ts.name) { + throw new TypeError("A time-series needs a name, e.g. \"accX\" or { name: \"accX\", unit: \"g\", quantity: \"acceleration\" }"); + } + return ts; +} + +/** + * The time-series entry of a dataset init request. Plain names stay plain + * strings, so servers without unit support keep accepting them. + */ +export function initEntry(ts) { + const { name, unit, quantity } = seriesSpec(ts); + if (!unit && !quantity) return name; + return { name, unit: unit || "", quantity: quantity || null }; +} + +/** + * The expected unit of every model input, from `manifest.input.units`. + * Exports made before units existed have none: every input is unknown. + * @returns {import("../index").InputUnit[]} + */ +export function manifestUnits(manifest) { + const names = manifest.input.timeseries; + const byName = {}; + for (const u of manifest.input.units || []) byName[u.name] = u; + return names.map((name) => { + const u = byName[name] || {}; + return { name, quantity: u.quantity || null, unit: u.unit || "", conversions: u.conversions || {} }; + }); +} + +const isUnknown = (input) => !input.quantity && !input.unit; + +/** The catalog unit id of `unit` within `quantityId`, matching ids, labels and aliases. */ +function catalogUnitId(catalog, quantityId, unit) { + const quantity = catalog && (catalog.quantities || []).find((q) => q.id === quantityId); + if (!quantity) return null; + const match = quantity.units.find((u) => u.id === unit || u.label === unit || (u.aliases || []).includes(unit)); + return match ? match.id : null; +} + +/** + * Works out how to convert each input from the unit the app delivers to the + * unit the model expects. + * + * @param {import("../index").InputUnit[]} inputs - from `manifestUnits` + * @param {{ [sensor: string]: string }} sensorUnits - unit each sensor delivers; sensors the model does not use are ignored + * @param {import("../index").UnitCatalog} [catalog] - optional, to also accept unit labels and aliases (e.g. "°C") + * @returns {({ scale: number, offset: number } | null)[]} per input, null = pass values through + */ +export function resolveConversions(inputs, sensorUnits, catalog) { + return inputs.map((input) => { + const unit = sensorUnits ? sensorUnits[input.name] : undefined; + if (unit === undefined || unit === null) return null; + if (typeof unit !== "string") throw new TypeError(`The unit of '${input.name}' must be a string`); + // trained before units existed, or on data of unknown unit: values go in as they are + if (isUnknown(input)) return null; + + let conv = input.conversions[unit]; + if (!conv && unit !== input.unit && input.quantity) { + const id = catalogUnitId(catalog, input.quantity, unit); + if (id) conv = input.conversions[id] || (id === input.unit ? { scale: 1, offset: 0 } : undefined); + } + if (!conv && unit === input.unit) conv = { scale: 1, offset: 0 }; + if (!conv) { + const accepted = Object.keys(input.conversions); + const expected = `${input.unit || "no unit"}${input.quantity ? ` (${input.quantity})` : ""}`; + throw new UnitError( + `'${input.name}' is declared in '${unit}', but the model expects ${expected} and cannot convert from '${unit}'. ` + + (accepted.length > 1 ? `Accepted units: ${accepted.join(", ")}.` : `Deliver it in ${input.unit || "the unit it was trained on"}.`) + ); + } + return conv.scale === 1 && conv.offset === 0 ? null : { scale: conv.scale, offset: conv.offset }; + }); +}